I'm trying to write a neural network that only requires the user to specify the dimensionality of the network. Concretely, the user might define a network like this:
nn = NN([2, 10, 20, 15, 2]) # 2 input, 2 output, 10 in hidden 1, 20 in hidden 2...
To do this, I'm trying to adapt some basic code. Please let me know what improvements can be made to improve readability (for example, I've considered cleaning up the distinction between wDims
(weight dimensions) and layer dims
because these variables seem redundant).
I would also appreciate any tips on how to implement a neural network as a graph. I've tried this but can't agree on what classes need to be defined or even how the graph would be stored (as a python dictionary?). Basically, some suggestions relating to good representation would be much appreciated.
First, auxiliary methods the network uses:
import random, math
random.seed(0)
def r_matrix(m, n, a = -0.5, b = 0.5):
return [[random.uniform(a,b) for j in range(n)] for i in range(m)]
def sigmoid(x):
return 1.0/ (1.0 + math.exp(-x))
def d_sigmoid(y):
return y * (1.0 - y)
Definition and construction of the network:
class NN:
def __init__(self, dims):
self.dims = dims
self.nO = self.dims[-1]
self.nI = self.dims[0]
self.nLayers = len(self.dims)
self.wDims = [ (self.dims[i-1], self.dims[i])\
for i in range(1, self.nLayers) ]
self.nWeights = len(self.wDims)
self.__initNeurons()
self.__initWeights()
def __initWeights(self):
self.weights = [0.0] * self.nWeights
for i in range(self.nWeights):
n_in, n_out = self.wDims[i]
self.weights[i] = r_matrix(n_in, n_out)
def __initNeurons(self):
self.layers = [0.0] * self.nLayers
for i in range(self.nLayers):
self.layers[i] = [0.0] * self.dims[i]
Implementation of back propagation and forward propagation:
def __activateLayer(self, i):
prev = self.layers[i-1]
n_in, n_out = self.dims[i-1], self.dims[i]
for j in range(n_out):
total = 0.0
for k in range(n_in):
total += prev[k] * self.weights[i-1][k][j] # num weights is always one less than num layers
self.layers[i][j] = sigmoid(total)
def __backProp(self, i, delta):
n_out, n_in = self.dims[i], self.dims[i+1]
next_delta = [0.0] * n_out
for j in range(n_out):
error = 0.0
for k in range(n_in):
error += delta[k] * self.weights[i][j][k]
pred = self.layers[i][j]
next_delta[j] = d_sigmoid(pred) * error
return next_delta
def __updateWeights(self, i, delta, alpha = .7):
n_in, n_out = self.wDims[i]
for j in range(n_in):
for k in range(n_out):
change = delta[k] * self.layers[i][j]
self.weights[i][j][k] += alpha * change
def feedForward(self, x):
if len(x) != self.nI:
raise ValueError('length of x must be same as num input units')
for i in range(self.nI):
self.layers[0][i] = x[i]
for i in range(1, self.nLayers):
self.__activateLayer(i)
def backPropLearn(self, y):
if len(y) != self.nO:
raise ValueError('length of y must be same as num output units')
delta_list = []
delta = [0.0] * self.nO
for k in range(self.nO):
pred = self.layers[-1][k]
error = y[k] - pred
delta[k] = d_sigmoid(pred) * error
delta_list.append(delta)
for i in reversed(range(1, self.nLayers-1)):
next_delta = self.__backProp(i, delta)
delta = next_delta
delta_list = [delta] + delta_list
# now perform the update
for i in range(self.nWeights):
self.__updateWeights(i, delta_list[i])
Predict and train methods. Yes I'm planning on improving train
to allow the user to specify the maximum number of iterations and alpha:
def predict(self, x):
self.feedForward(x)
return self.layers[-1]
def train(self, T):
i, MAX = 0, 5000
while i < MAX:
for t in T:
x, y = t
self.feedForward(x)
self.backPropLearn(y)
i += 1
Sample data. Teach the network the numbers 0 through 4:
# no. 0
t0 = [ 0,1,1,1,0,\
0,1,0,1,0,\
0,1,0,1,0,\
0,1,0,1,0,\
0,1,1,1,0 ]
# no. 1
t1 = [ 0,1,1,0,0,\
0,0,1,0,0,\
0,0,1,0,0,\
0,0,1,0,0,\
1,1,1,1,1 ]
# no. 2
t2 = [ 0,1,1,1,0,\
0,0,0,1,0,\
0,1,1,1,0,\
0,1,0,0,0,\
0,1,1,1,0 ]
# no. 3
t3 = [ 0,1,1,1,0,\
0,0,0,1,0,\
0,1,1,1,0,\
0,0,0,1,0,\
0,1,1,1,0 ]
# no. 4
t4 = [ 0,1,0,1,0,\
0,1,0,1,0,\
0,1,1,1,0,\
0,0,0,1,0,\
0,0,0,1,0 ]
T = [(t0, [1,0,0,0,0]), (t1, [0,1,0,0,0]), (t2, [0,0,1,0,0]), (t3, [0,0,0,1,0]), (t4, [0,0,0,0,1])]
nn = NN([25, 50, 50, 20, 5])
nn.train(T)