gradient descent implementation

Question

I am attempting to implement my neural net for a fully connected network configuration. I don't know much about neural networks and ai, so the implementation is not great and may contain errors however it has been tested and seems to give the desired results.

Full code:

import numpy as np
import pandas as pd

X = np.array([[0.50, 1.00, 0.75, 1], [1.00, 0.50, 0.75, 1])

t = np.array([[1, 0], [1, 0]])

learning_rate = 0.1

# Initialize weights for the hidden layer (small random)
weights_hidden = np.array([[0.74, 0.13, 0.68], [0.8, 0.4, 0.10]]) # Last set of weights are for the bias

# Initialize weights for the output layer (small random)
weights_output = np.array([[0.35, 0.8], [0.50, 0.13], [0.90, 0.8]]) # Last set of weights are for the bias

def sigmod(x):
    return 1 / (1+np.exp(-x))

def mean_square_error(y, t):
    return ((y - t)**2).sum() / (2*y.size)

if __name__ == "__main__":
    iterations = 2000
    
    # Train the model
    for epoch in range(0, iterations):
        for episode in range(0, len(X)):

            # feed forward on the hidden layer
            A1 = np.dot([X[episode]], weights_hidden)
            B1 = sigmod(A1)
            
            # feed forward on the output layer (no activation function needed)
            B2 = np.dot(np.append(B1, 1), weights_output) # Append 1 for bias
            
            # Backpropagation
            # Error on the output layer
            BP1 = t[episode] - B2
            #print(f"Output Layer Error : {BP1}")
            
            # Error on the hidden layer
            BP2_1 = B1[0][0] * (1 - B1[0][0]) * ((BP1[0] * weights_output[0][0]) + (BP1[1] * weights_output[0][1]))
            
            # Find our weight changes for hidden layer
            weights_hidden_update = np.array([])
            
            for input in [X[episode]]:
                for error in BP2:
                    weights_hidden_update = np.append(weights_hidden_update, learning_rate * error * input)
            
            # Convert the 1d array to 2d
            weights_hidden_update = np.reshape(weights_hidden_update, (-1, 3))
            
            # Find our weight changes for output layer
            weights_output_update = np.array([])
            
            for input in np.append(B1, 1):
                for error in BP1:
                    weights_output_update = np.append(weights_output_update, learning_rate * error * input)
                    
            weights_output_update = np.reshape(weights_output_update, (-1, 2))
            weights_output = weights_output + weights_output_update
            weights_hidden = weights_hidden + weights_hidden_update

Answer 1

Your array literal formatting could be improved - try to line up rows and values.

Add PEP484 type hints.

Your __main__ code needs to be moved into functions. As it is, all of that code is still in the global namespace. You should also remove any mutating state out of global scope and pass it around in parameters.

Successive DataFrame.append() is non-ideal for at least two reasons: it's slow, and it produces a deluge of deprecation warnings. The faster method is to simply build up a list and then convert it to a data frame at the end.

for input in [X[episode]] makes no sense and you can simply assign input = X[episode].

Your inner loops to calculate the _update variables need to go away and be replaced by vectorised broadcast expressions. This will also obviate the reshape.

Your calculation for BP2 should likewise be vectorised and not split into three elements.

Your error progression graph is unhelpful. First, it's crucial that it be semilog-y, since your error goes so low. Second, you'll want to use an aggregating plotter that shows confidence intervals since the error has high variance and is very dense data. Seaborn does this automatically.

Suggested

from typing import Sequence

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns

# features
X = np.array([
    [ 0.50,  1.00, 0.75, 1],
    [ 1.00,  0.50, 0.75, 1],
    [ 1.00,  1.00, 1.00, 1],
    [-0.01,  0.50, 0.25, 1],
    [ 0.50, -0.25, 0.13, 1],
    [ 0.01,  0.02, 0.05, 1],
])


LEARNING_RATE = 0.1


def sigmod(x: np.ndarray) -> np.ndarray:
    return 1 / (1 + np.exp(-x))


# Compute softmax values for each sets of scores in x
def soft_max(x: np.ndarray) -> np.ndarray:
    return np.exp(x) / np.sum(np.exp(x), axis=0)


def mean_square_error(y: np.ndarray, t: np.ndarray) -> np.ndarray:
    return ((y - t) ** 2).sum() / (2 * y.size)


def print_weights(weights_hidden: np.ndarray, weights_output: np.ndarray) -> None:
    print(f"Weights_hidden: {weights_hidden}\n")
    print(f"Weights_output: {weights_output}\n")


def test_model(input: Sequence[int], weights_hidden: np.ndarray, weights_output: np.ndarray) -> None:
    X = np.array([input])
    A1 = np.dot(X, weights_hidden)
    B1 = sigmod(A1)
    B2 = np.dot(np.append(B1, 1), weights_output)

    print(f"Input: {X}")
    print(f"Output: {B2}")  # Output for the test data
    print(f"Softmax (Probability distribution) {soft_max(B2)}\n")  # Output probability distribution the test data


def train_iterate(
    episode: int,
    epoch: int,
    weights_hidden: np.ndarray,
    weights_output: np.ndarray,
    results: list[dict],
    t: np.ndarray,
) -> tuple[
    np.ndarray,  # hidden update
    np.ndarray,  # output update
]:
    # feed forward on the hidden layer
    A1 = np.dot([X[episode]], weights_hidden)
    B1 = sigmod(A1)

    # feed forward on the output layer (no activation function needed)
    B2 = np.dot(np.append(B1, 1), weights_output)  # Append 1 for bias

    # Get the error
    mean_square_error_rate = mean_square_error(B2, t[episode])

    # Add the error and epoch to the results dataframe (for analysis/plotting)
    results.append({"mse": mean_square_error_rate, "epochs": epoch})

    # Backpropagation
    # Error on the output layer
    BP1 = t[episode] - B2

    # Error on the hidden layer
    BP2, = B1 * (1 - B1) * (BP1 * weights_output[:-1,:]).sum(axis=1)

    # Find our weight changes for hidden layer
    input = X[episode]
    weights_hidden_update = LEARNING_RATE * BP2[np.newaxis, :] * input[:, np.newaxis]

    # Find our weight changes for output layer
    weights_output_update = LEARNING_RATE * BP1[np.newaxis, :] * np.append(B1, 1)[:, np.newaxis]

    return weights_output_update, weights_hidden_update


def train() -> tuple[
    np.ndarray,  # hidden
    np.ndarray,  # output
    pd.DataFrame,  # results
]:
    # Initialize weights for the hidden layer (small random)
    weights_hidden = np.array([
        [0.74, 0.13, 0.68],
        [0.80, 0.40, 0.10],
        [0.35, 0.97, 0.96],
        [0.90, 0.45, 0.36],
    ])  # Last set of weights are for the bias

    # Initialize weights for the output layer (small random)
    weights_output = np.array([
        [0.35, 0.80],
        [0.50, 0.13],
        [0.90, 0.80],
        [0.98, 0.92],
    ])  # Last set of weights are for the bias

    results = []

    # Targets
    t = np.array([
        [1, 0],
        [1, 0],
        [1, 0],
        [0, 1],
        [0, 1],
        [0, 1],
    ])

    iterations = 50

    # Train the model
    for epoch in range(iterations):
        for episode in range(len(X)):
            weights_output_update, weights_hidden_update = train_iterate(
                episode, epoch, weights_hidden, weights_output, results, t
            )

            # Update our output weights
            weights_output += weights_output_update
            weights_hidden += weights_hidden_update

    results = pd.DataFrame.from_records(results)
    return weights_hidden, weights_output, results


def test_model_cases(
    weights_hidden: np.ndarray,
    weights_output: np.ndarray,
) -> None:
    test_model(( 0.50,  1.00, 0.75, 1), weights_hidden, weights_output)  # Expected output: 1 0
    test_model(( 1.00,  0.50, 0.75, 1), weights_hidden, weights_output)  # Expected output: 1 0
    test_model(( 1.00,  1.00, 1.00, 1), weights_hidden, weights_output)  # Expected output: 1 0
    test_model((-0.01,  0.50, 0.25, 1), weights_hidden, weights_output)  # Expected output: 0 1
    test_model(( 0.50, -0.25, 0.13, 1), weights_hidden, weights_output)  # Expected output: 0 1
    test_model(( 0.01,  0.02, 0.05, 1), weights_hidden, weights_output)  # Expected output: 0 1
    test_model(( 0.30,  0.70, 0.90, 1), weights_hidden, weights_output)


def plot_progress(results: Sequence[dict]) -> None:
    # Plot the results (MSE and epochs)
    fig, ax = plt.subplots()
    sns.lineplot(data=results, x='epochs', y='mse', ax=ax)
    ax: plt.Axes
    ax.set_title("Mean Squared Error")
    ax.set_xlabel("Epochs")
    ax.set_ylabel("MSE")
    ax.set_yscale('log')
    plt.show()  # Show the plot


def main() -> None:
    weights_hidden, weights_output, results = train()

    test_model_cases(weights_hidden, weights_output)

    plot_progress(results)


if __name__ == "__main__":
    main()