I am attempting to implement my neural net for a fully connected network configuration. I don't know much about neural networks and ai, so the implementation is not great and may contain errors however it has been tested and seems to give the desired results.

Full code:

import numpy as np
import pandas as pd

X = np.array([[0.50, 1.00, 0.75, 1], [1.00, 0.50, 0.75, 1])

t = np.array([[1, 0], [1, 0]])

learning_rate = 0.1

# Initialize weights for the hidden layer (small random)
weights_hidden = np.array([[0.74, 0.13, 0.68], [0.8, 0.4, 0.10]]) # Last set of weights are for the bias

# Initialize weights for the output layer (small random)
weights_output = np.array([[0.35, 0.8], [0.50, 0.13], [0.90, 0.8]]) # Last set of weights are for the bias

def sigmod(x):
return 1 / (1+np.exp(-x))

def mean_square_error(y, t):
return ((y - t)**2).sum() / (2*y.size)

if __name__ == "__main__":
iterations = 2000

# Train the model
for epoch in range(0, iterations):
for episode in range(0, len(X)):

# feed forward on the hidden layer
A1 = np.dot([X[episode]], weights_hidden)
B1 = sigmod(A1)

# feed forward on the output layer (no activation function needed)
B2 = np.dot(np.append(B1, 1), weights_output) # Append 1 for bias

# Backpropagation
# Error on the output layer
BP1 = t[episode] - B2
#print(f"Output Layer Error : {BP1}")

# Error on the hidden layer
BP2_1 = B1[0][0] * (1 - B1[0][0]) * ((BP1[0] * weights_output[0][0]) + (BP1[1] * weights_output[0][1]))

# Find our weight changes for hidden layer
weights_hidden_update = np.array([])

for input in [X[episode]]:
for error in BP2:
weights_hidden_update = np.append(weights_hidden_update, learning_rate * error * input)

# Convert the 1d array to 2d
weights_hidden_update = np.reshape(weights_hidden_update, (-1, 3))

# Find our weight changes for output layer
weights_output_update = np.array([])

for input in np.append(B1, 1):
for error in BP1:
weights_output_update = np.append(weights_output_update, learning_rate * error * input)

weights_output_update = np.reshape(weights_output_update, (-1, 2))
weights_output = weights_output + weights_output_update
weights_hidden = weights_hidden + weights_hidden_update

• It's refreshing to see this attempted from first principles instead of with one of the go-to libraries. Commented Mar 14, 2022 at 22:06

Your array literal formatting could be improved - try to line up rows and values.

Your __main__ code needs to be moved into functions. As it is, all of that code is still in the global namespace. You should also remove any mutating state out of global scope and pass it around in parameters.

Successive DataFrame.append() is non-ideal for at least two reasons: it's slow, and it produces a deluge of deprecation warnings. The faster method is to simply build up a list and then convert it to a data frame at the end.

for input in [X[episode]] makes no sense and you can simply assign input = X[episode].

Your inner loops to calculate the _update variables need to go away and be replaced by vectorised broadcast expressions. This will also obviate the reshape.

Your calculation for BP2 should likewise be vectorised and not split into three elements.

Your error progression graph is unhelpful. First, it's crucial that it be semilog-y, since your error goes so low. Second, you'll want to use an aggregating plotter that shows confidence intervals since the error has high variance and is very dense data. Seaborn does this automatically.

## Suggested

from typing import Sequence

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns

# features
X = np.array([
[ 0.50,  1.00, 0.75, 1],
[ 1.00,  0.50, 0.75, 1],
[ 1.00,  1.00, 1.00, 1],
[-0.01,  0.50, 0.25, 1],
[ 0.50, -0.25, 0.13, 1],
[ 0.01,  0.02, 0.05, 1],
])

LEARNING_RATE = 0.1

def sigmod(x: np.ndarray) -> np.ndarray:
return 1 / (1 + np.exp(-x))

# Compute softmax values for each sets of scores in x
def soft_max(x: np.ndarray) -> np.ndarray:
return np.exp(x) / np.sum(np.exp(x), axis=0)

def mean_square_error(y: np.ndarray, t: np.ndarray) -> np.ndarray:
return ((y - t) ** 2).sum() / (2 * y.size)

def print_weights(weights_hidden: np.ndarray, weights_output: np.ndarray) -> None:
print(f"Weights_hidden: {weights_hidden}\n")
print(f"Weights_output: {weights_output}\n")

def test_model(input: Sequence[int], weights_hidden: np.ndarray, weights_output: np.ndarray) -> None:
X = np.array([input])
A1 = np.dot(X, weights_hidden)
B1 = sigmod(A1)
B2 = np.dot(np.append(B1, 1), weights_output)

print(f"Input: {X}")
print(f"Output: {B2}")  # Output for the test data
print(f"Softmax (Probability distribution) {soft_max(B2)}\n")  # Output probability distribution the test data

def train_iterate(
episode: int,
epoch: int,
weights_hidden: np.ndarray,
weights_output: np.ndarray,
results: list[dict],
t: np.ndarray,
) -> tuple[
np.ndarray,  # hidden update
np.ndarray,  # output update
]:
# feed forward on the hidden layer
A1 = np.dot([X[episode]], weights_hidden)
B1 = sigmod(A1)

# feed forward on the output layer (no activation function needed)
B2 = np.dot(np.append(B1, 1), weights_output)  # Append 1 for bias

# Get the error
mean_square_error_rate = mean_square_error(B2, t[episode])

# Add the error and epoch to the results dataframe (for analysis/plotting)
results.append({"mse": mean_square_error_rate, "epochs": epoch})

# Backpropagation
# Error on the output layer
BP1 = t[episode] - B2

# Error on the hidden layer
BP2, = B1 * (1 - B1) * (BP1 * weights_output[:-1,:]).sum(axis=1)

# Find our weight changes for hidden layer
input = X[episode]
weights_hidden_update = LEARNING_RATE * BP2[np.newaxis, :] * input[:, np.newaxis]

# Find our weight changes for output layer
weights_output_update = LEARNING_RATE * BP1[np.newaxis, :] * np.append(B1, 1)[:, np.newaxis]

return weights_output_update, weights_hidden_update

def train() -> tuple[
np.ndarray,  # hidden
np.ndarray,  # output
pd.DataFrame,  # results
]:
# Initialize weights for the hidden layer (small random)
weights_hidden = np.array([
[0.74, 0.13, 0.68],
[0.80, 0.40, 0.10],
[0.35, 0.97, 0.96],
[0.90, 0.45, 0.36],
])  # Last set of weights are for the bias

# Initialize weights for the output layer (small random)
weights_output = np.array([
[0.35, 0.80],
[0.50, 0.13],
[0.90, 0.80],
[0.98, 0.92],
])  # Last set of weights are for the bias

results = []

# Targets
t = np.array([
[1, 0],
[1, 0],
[1, 0],
[0, 1],
[0, 1],
[0, 1],
])

iterations = 50

# Train the model
for epoch in range(iterations):
for episode in range(len(X)):
weights_output_update, weights_hidden_update = train_iterate(
episode, epoch, weights_hidden, weights_output, results, t
)

# Update our output weights
weights_output += weights_output_update
weights_hidden += weights_hidden_update

results = pd.DataFrame.from_records(results)
return weights_hidden, weights_output, results

def test_model_cases(
weights_hidden: np.ndarray,
weights_output: np.ndarray,
) -> None:
test_model(( 0.50,  1.00, 0.75, 1), weights_hidden, weights_output)  # Expected output: 1 0
test_model(( 1.00,  0.50, 0.75, 1), weights_hidden, weights_output)  # Expected output: 1 0
test_model(( 1.00,  1.00, 1.00, 1), weights_hidden, weights_output)  # Expected output: 1 0
test_model((-0.01,  0.50, 0.25, 1), weights_hidden, weights_output)  # Expected output: 0 1
test_model(( 0.50, -0.25, 0.13, 1), weights_hidden, weights_output)  # Expected output: 0 1
test_model(( 0.01,  0.02, 0.05, 1), weights_hidden, weights_output)  # Expected output: 0 1
test_model(( 0.30,  0.70, 0.90, 1), weights_hidden, weights_output)

def plot_progress(results: Sequence[dict]) -> None:
# Plot the results (MSE and epochs)
fig, ax = plt.subplots()
sns.lineplot(data=results, x='epochs', y='mse', ax=ax)
ax: plt.Axes
ax.set_title("Mean Squared Error")
ax.set_xlabel("Epochs")
ax.set_ylabel("MSE")
ax.set_yscale('log')
plt.show()  # Show the plot

def main() -> None:
weights_hidden, weights_output, results = train()

test_model_cases(weights_hidden, weights_output)

plot_progress(results)

if __name__ == "__main__":
main()


• The Sequence is to indicate that the function accepts a list or a tuple - roughly, anything indexable with an order and a length. So yes, it's to show what types the function takes and outputs, but it isn't only for display: if you run mypy, it will tell you if something looks wrong based on type hints. Commented Mar 15, 2022 at 12:41
• By 'fast' do you mean that the error decreases in fewer iterations, or that the training completes in less time? If the former, it may be because I fixed an error where you were reshaping in the wrong order. If the latter, it's probably attributable to vectorisation. Commented Mar 15, 2022 at 12:43
• thank you, yeah probably was slow due to my bad code. I was wondering is there a way I can turn your code into a non numpy version, so just using the math library?
– Mj _
Commented Mar 17, 2022 at 20:12
• "Can" and "should" are two different things. Yes you can; no you absolutely shouldn't. Commented Mar 17, 2022 at 21:51
• Do you think you could show me how, if if its not practical so I can compare the 2 please
– Mj _
Commented Mar 17, 2022 at 22:01