I've implemented a bunch of activation functions for neural networks, and I just want have validation that they work correctly mathematically. I implemented sigmoid, tanh, relu, arctan, step function, squash, and gaussian and I use their implicit derivative (in terms of the output) for backpropagation.
import numpy as np
def sigmoid(x, derivative=False):
if (derivative == True):
return x * (1 - x)
return 1 / (1 + np.exp(-x))
def tanh(x, derivative=False):
if (derivative == True):
return (1 - (x ** 2))
return np.tanh(x)
def relu(x, derivative=False):
if (derivative == True):
for i in range(0, len(x)):
for k in range(len(x[i])):
if x[i][k] > 0:
x[i][k] = 1
else:
x[i][k] = 0
return x
for i in range(0, len(x)):
for k in range(0, len(x[i])):
if x[i][k] > 0:
pass # do nothing since it would be effectively replacing x with x
else:
x[i][k] = 0
return x
def arctan(x, derivative=False):
if (derivative == True):
return (np.cos(x) ** 2)
return np.arctan(x)
def step(x, derivative=False):
if (derivative == True):
for i in range(0, len(x)):
for k in range(len(x[i])):
if x[i][k] > 0:
x[i][k] = 0
return x
for i in range(0, len(x)):
for k in range(0, len(x[i])):
if x[i][k] > 0:
x[i][k] = 1
else:
x[i][k] = 0
return x
def squash(x, derivative=False):
if (derivative == True):
for i in range(0, len(x)):
for k in range(0, len(x[i])):
if x[i][k] > 0:
x[i][k] = (x[i][k]) / (1 + x[i][k])
else:
x[i][k] = (x[i][k]) / (1 - x[i][k])
return x
for i in range(0, len(x)):
for k in range(0, len(x[i])):
x[i][k] = (x[i][k]) / (1 + abs(x[i][k]))
return x
def gaussian(x, derivative=False):
if (derivative == True):
for i in range(0, len(x)):
for k in range(0, len(x[i])):
x[i][k] = -2* x[i][k] * np.exp(-x[i][k] ** 2)
for i in range(0, len(x)):
for k in range(0, len(x[i])):
x[i][k] = np.exp(-x[i][k] ** 2)
return x
The input x
is the result of the output matrix of the previous layer multiplied by the weight matrix. The output x
is the resultant of the activation function on the matrix to be used for the next layer.
Did I implement them in the way that it was intended to be used, or do some of these not give the intended results?
Here is the gradient descent algorithm I implemented:
import numpy as np
from Neural_Network import Activation_Function as af
def GD(layers, weights, proper, acti):
deltas = []
llayerError = layers[-1] - proper
llayerDelta = llayerError * acti(layers[-1], True)
deltas.append(llayerDelta)
for i in range(-1, -len(layers), -1):
layerError = deltas[abs((i + 1))].dot(weights[i].T)
layerDelta = layerError * acti(layers[i-1], True)
deltas.append(layerDelta)
return deltas