A simple neural network I wrote in Python without libraries. I avoided implementing it in matrix form because I sought to get a basic understanding of the way NN's work first. For that reason I'm strongly favoring legibility over efficiency. I tried to keep my code readable and pyhonic, any style feedback would be particularly appreciated.
A quirk about this design is it does back propagation on a per training example basis and uses momentum to try and avoid over fitting to specific examples. Also I realized I never added base values to the neurons, it seems to work alright with out them but if anyone has a more in depth understanding of why you'd want them I'd be curious to hear about that.
import math
import random
import data
def sigmoid(x):
return 1 / (1 + math.exp(-x))
def sigmoid_prime(x):
return x * (1.0 - x)
def loss(x,y):
return sum([(a-b)**2 for (a,b) in zip(x,y)])
class Neuron():
learning_rate = 0.015
momentum_loss = 0.03
def __init__(self, input_neurons):
self.weights = [random.uniform(-1,1) for _ in range(input_neurons)]
self.momentum = [0 for _ in range(input_neurons)]
def forward(self, inputs):
dot = sum([x*y for (x,y) in zip(inputs, self.weights)])
self.output = sigmoid(dot)
return self.output
def backpropagate(self, inputs, error):
error_values = list()
gradient = error * sigmoid_prime(self.output)
for i, inp in enumerate(inputs):
self.nudge_weight(i, gradient * inp)
error_values.append(self.weights[i] * gradient)
return error_values
def nudge_weight(self, weight, amount):
change = amount * Neuron.learning_rate
self.momentum[weight] += change
self.momentum[weight] *= (1 - Neuron.momentum_loss)
self.weights[weight] += change + self.momentum[weight]
class Network():
def __init__(self, topology):
self.layers = list()
for i in range(1,len(topology)):
self.layers.append([Neuron(topology[i-1]) for _ in range(topology[i])])
def forward(self, data):
output = data
for layer in self.layers:
output = [neuron.forward(output) for neuron in layer]
return output
def backpropagate(self, data, output, target):
error_values = [tval - output for (tval, output) in zip(target, output)]
for i in range(len(self.layers)-1,0,-1):
layer_output = [neuron.output for neuron in self.layers[i-1]]
error_values = self.backpropagate_layer(i, error_values, layer_output)
self.backpropagate_layer(0, error_values, data)
def backpropagate_layer(self, layer, error_values, inputs):
next_errors = list()
for neuron, error in zip(self.layers[layer], error_values):
bp_error = neuron.backpropagate(inputs,error)
if not next_errors:
next_errors = bp_error
else:
next_errors = [a+b for a,b in zip(next_errors,bp_error)]
return next_errors
The full source code for project including the data base and some other testing code can be found here: https://github.com/RowanL3/Neural-Network
