1
\$\begingroup\$

My neural network is buggy somewhere. However, the reason I am posting here and not Stack Overflow is because a buggy neural network can still be trained to some degree and will compile/perform better than no neural network, unlike other programs. These "bugs" can be solved by optimizations/better initialization of the weight values/better implementation of back-propagation (it is not completely wrong).

No neural network can be correct 100% (there is always something I can change to make it have a lower cost). The network does not have a binary output or certain number it must hit, rather, it must simply reduce the cost after each epoch, which it currently does. Asking for reviews regarding how how I can make it reduce the cost further does not imply that my program is not currently working. Rather, it implies that it is not most optimal, which is present among other questions as well (as other questions ask how programs can be more performant, for instance).

Since my neural network is from scratch (and thus pretty long for the casual reviewer), I would like two specific functions to be reviewed that are causing me problems:

  1. offsetLayerIndex

    Since there are no weights for a neural network in the first layer, I offset all the layer indices of my stored weights by -1 when writing to the weight storage. However, changing the offset to 0 (none) changed the program, even though every where I write to weight storage I offset the index.

  2. updateParameterGradients

    This is the backpropagation implementation which I based off these 4 formulas from here:

    enter image description here

Finally, the printing in the #if -> #endif printing macros is the least important (just for convenience), and thus doesn't need to be reviewed.

functions.h

#ifndef NEURALNETWORK_FUNCTIONS_H
#define NEURALNETWORK_FUNCTIONS_H

/**
 * Activates the value given by the weighted sum function. This activated value will be put
 * into the neuron.
 *
 * @param weightedSum The result of the weighted sum function.
 * @return The activated neuron value.
 */
double getDefaultActivation(double weightedSum);

/**
 * Calculates the derivative of the sigmoid function at the weighted sum, which, when inputted into sigmoid,
 * gives activationValue.
 *
 * @param activationValue Since the derivative of a sigmoid does not depend on the input of a sigmoid,
 * but rather the output of the sigmoid, the value of the neuron is passed for efficiency.
 * @return activationValue * (1 - activationValue)
 */
double getDefaultActivationDerivative(double activationValue);

/**
 * Calculates the derivative of the function that produces the weighted sum given a weight, neuron value and bias,
 * with respect to the neuron value.
 *
 * @param weight The weight that the function multiplies with the neuron value, which is w[endLayerIndex, endNeuronIndex,
 * startNeuronIndex]
 * @return Δ Z[endLayerIndex, endNeuronIndex] / Δ a[startLayerIndex, startNeuronIndex]
 */
double getWeightedSumNeuronValueDerivative(double weight);

/**
 * Calculates the derivative of the function that produces the weighted sum given a weight, neuron value and bias,
 * with respect to the weight.
 *
 * @param neuronValue The neuron value that the function multiplies with the weight, which is a[startLayerIndex,
 * startNeuronIndex]
 * @return Δ Z[endLayerIndex, endNeuronIndex] / Δ w[endLayerIndex, endNeuronIndex]
 */
double getWeightedSumWeightDerivative(double neuronValue);

/**
 * Calculates the derivative of the function that produces the weighted sum given a weight, neuron value and bias,
 * with respect to the bias.
 *
 * @return Δ Z[endLayerIndex, endNeuronIndex] / Δ b[endLayerIndex, endNeuronIndex]
 */
double getWeightedSumBiasDerivative();

/**
 * Calculates the cost of {@code neuronValue} relative to the target output.
 *
 * @param neuronValue The output produced by the neural network.
 * @param intendedValue The correct output that the neural network aims to produce.
 * @return The "cost"/effect of the network outputting {@code neuronValue} instead of {@code intendedValue}.
 */
double getDefaultCost(double neuronValue, double intendedValue);

/**
 * Calculates the derivative of the cost of {@code neuronValue} relative to the target output.
 *
 * @param neuronValue The output produced by the neural network.
 * @param intendedValue The correct output that the neural network aims to produce.
 * @return Δ "cost"/effect of the network outputting {@code neuronValue} instead of {@code intendedValue} /
 *         Δ neuronValue
 */
double getDefaultCostDerivative(double neuronValue, double intendedValue);

/**
 * Calculates the initial value that a weight[endLayerIndex][endNeuronIndex][startNeuronIndex]
 * should have before the first epoch of the network.
 *
 * @param previousLayerSize Number of neurons in startLayerIndex.
 * @param layerSize Number on neurons in endLayerIndex.
 * @return
 */
double getDefaultInitialWeightValue(double previousLayerSize, double layerSize);

/**
 * Calculates the initial value that a bias[endLayerIndex][endNeuronIndex]
 * should have before the first epoch of the network.
 *
 * @param previousLayerSize Number of neurons in startLayerIndex.
 * @param layerSize Number on neurons in endLayerIndex.
 * @return
 */
double getDefaultInitialBiasValue(double previousLayerSize, double layerSize);

#endif

functions.c

#include "functions.h"
#include "math.h"

double getDefaultActivation(double weightedSum) {
    double eToWSum = pow(M_E, weightedSum);
    return eToWSum / (eToWSum + 1);
}

double getDefaultActivationDerivative(double activationValue) {
    return activationValue * (1 - activationValue);
}

/**
 * start_index can be any value
 *
 * @param neuronValue value of w[column][end_index][start_index]
 * @return delta z[column, end_index] / delta w[column, end_index, start_index]. This is always
 * equivalent to the neuron value itself since weight is multiplied with the neuron value when calculating
 * weighted sum.
 */
double getWeightedSumWeightDerivative(double neuronValue) {
    return neuronValue;
}

double getWeightedSumNeuronValueDerivative(double weight) {
    return weight;
}

double getDefaultInitialWeightValue(double previousLayerSize, double layerSize) {
    return sqrt(2 / (previousLayerSize));
}

double getDefaultInitialBiasValue(double previousLayerSize, double layerSize) {
    return 0;
}


/**
 * @return delta z[column, end_index] / delta b[column, end_index]. This is always
 * constant (1) since the bias is not multiplied by the neuron value when calculating weighted sum.
 */
double getWeightedSumBiasDerivative() {
    return 1;
}

double getDefaultCost(double neuronValue, double intendedValue) {
    double difference = neuronValue - intendedValue;

    return pow(difference, 2);
}
double getDefaultCostDerivative(double neuronValue, double intendedValue) {
    return neuronValue - intendedValue;
}

model.h

#ifndef NEURALNETWORK_MODEL_H
#define NEURALNETWORK_MODEL_H

#include <stdbool.h>
#include "data.h"

#define NUMBER_OF_LAYERS 3
#define INPUT_LAYER 0
#define OUTPUT_LAYER (NUMBER_OF_LAYERS - 1)
#define NUMBER_OF_TRAINING_FEATURE_VECTORS 2

typedef double (*CostFunction)(double, double);
typedef double (*ActivationFunction)(double);
typedef double (*WeightInitializationFunction)(double, double);
typedef double (*BiasInitializingFunction)(double, double);

/**
 * Represents a model to be used in regards to training the vehicle to move efficiently.
 * Implemented as a neural network.
 */
struct Model {
    /* Weight of link between two neurons is received with indices
     * [layer index of end neuron][index of end neuron within its layer][index of start neuron within its layer] */
    double** weights[NUMBER_OF_LAYERS - 1];
    double* biases[NUMBER_OF_LAYERS - 1];
    double* values[NUMBER_OF_LAYERS];

    int neuronsPerLayer[NUMBER_OF_LAYERS];
    double learningRate;

    /**
     * the function used to activate the neuron. The value returned by this function is put into the neuron.
     */
    ActivationFunction getActivation;

    /**
     * the derivative of the function used to activate the neuron.
     */
    ActivationFunction getActivationChange;

    /**
     * the function uses to calculate the cost, given an output neuron's value and its target value
     */
    CostFunction getCost;

    /**
     * the function used to calculate the derivative of the cost with respect to the output neuron's value, given the
     * output neuron's value and its target value
     */
    CostFunction getCostDerivative;

    WeightInitializationFunction getInitialWeightValue;
    BiasInitializingFunction getInitialBiasValue;

    /**
     * specifies whether the given activation function derivative can take the output of the activation function
     * rather than the input. This is the case for sigmoid
     */
    bool activationFunctionDerivativeUsesOutput;
};

void train(struct Model* model, struct Data* data, int inputColumnIndices[], int outputColumnIndices[]);
void test(struct Model* model, struct Data* data, int inputColumnIndices[], int outputColumnIndices[], double* predictedOutputs[], double costs[]);
void compute(struct Model* model, struct Data* data, int inputColumnIndices[], double cost[]);
void initParameters(struct Model* model);
void initValues(struct Model* model);

int offsetLayer(int layerIndex);

void initInput(double input[], const double entry[], const int inputColumnIndices[], int inputColumnIndicesCount);
void initTargetOutput(double targetOutput[], const double entry[], const int targetOutputIndices[], int targetOutputIndicesCount);

#endif

model.c

#include <stdlib.h>
#include <stdio.h>
#include <memory.h>
#include "model.h"
#include "functions.h"

#define GRADIENT_CHECKING true
#define PRINT_NEURON_VALUE false
#define PRINT_WEIGHT_UPDATE false
#define PRINT_BIAS_UPDATE false
#define PRINT_EPOCH_UPDATE true

/**
 * No weights are in the INPUT_LAYER. Thus, layer N is indexed as N - 1 in the weights
 * array of matrices.
 * @param layerIndex The layer index within the neural network.
 * @return The index within the weights storage of the matrix of weights that belong to layers[layerIndes].
 */
int offsetLayer(int layerIndex) {
    return layerIndex - 1;
}

/**
 * @param model
 * @param input The head to ann input array of size <code>model.neuronsPerLayer[INPUT_LAYER]</code> that has the inputs
 * of the model.
 */
void setInput(struct Model* model, double* inputHead) {
    model->values[INPUT_LAYER] = inputHead;
}

void propagateInputForward(struct Model* model, double* inputHead) {
    setInput(model, inputHead);

    for (int endLayerIndex = 1; endLayerIndex < NUMBER_OF_LAYERS; endLayerIndex++) {
        int offsetEndLayerIndex = offsetLayer(endLayerIndex);
        int startLayerIndex = endLayerIndex - 1;

        int endNeuronCount = model->neuronsPerLayer[endLayerIndex];
        int startNeuronCount = model->neuronsPerLayer[startLayerIndex];

        for (int endNeuronIndex = 0; endNeuronIndex < endNeuronCount; endNeuronIndex++) {
            double weightedSum = 0.0;
            double bias = model->biases[offsetEndLayerIndex][endNeuronIndex];

            for (int startNeuronIndex = 0; startNeuronIndex < startNeuronCount; startNeuronIndex++) {
                double weightOfLink = model->weights[offsetEndLayerIndex][endNeuronIndex][startNeuronIndex];
                double previousNeuronValue = model->values[startLayerIndex][startNeuronIndex];

                double weightedInfluence = weightOfLink * previousNeuronValue + bias;
                weightedSum += weightedInfluence;
            }

            double activatedNeuronValue = model->getActivation(weightedSum);

#if PRINT_NEURON_VALUE
            printf("Neuron[%i][%i] - Pre-Acivation: %lf, Post-Activation: %lf\n", layerIndex, neuronIndex, weightedSum, activatedNeuronValue);
#endif
            model->values[endLayerIndex][endNeuronIndex] = activatedNeuronValue;
        }
    }
}

#if GRADIENT_CHECKING
double getTotalCost(struct Model* model, const double targetOutputs[]) {
    int outputNeuronCount = model->neuronsPerLayer[OUTPUT_LAYER];

    double totalCost = 0.0;

    for (int outputNeuronIndex = 0; outputNeuronIndex < outputNeuronCount; outputNeuronIndex++) {
        double outputNeuronValue = model->values[OUTPUT_LAYER][outputNeuronIndex];
        double expectedOutputNeuronValue = targetOutputs[outputNeuronIndex];

        double cost = model->getCost(outputNeuronValue, expectedOutputNeuronValue);
        totalCost += cost;
    }

    return totalCost;
}

void initCheckParameterGradients(struct Model* model, double** checkWeightGradients[], double* checkBiasGradients[]) {
    for (int endLayerIndex = 1; endLayerIndex < NUMBER_OF_LAYERS; endLayerIndex++) {
        int offsetEndLayerIndex = offsetLayer(endLayerIndex);
        int startLayerIndex = endLayerIndex - 1;

        int endNeuronCount = model->neuronsPerLayer[endLayerIndex];
        int startNeuronCount = model->neuronsPerLayer[startLayerIndex];
        checkWeightGradients[offsetEndLayerIndex] = malloc(sizeof(double*) * endNeuronCount);
        checkBiasGradients[offsetEndLayerIndex] = malloc(sizeof(double) * endNeuronCount);

        for (int endNeuronIndex = 0; endNeuronIndex < endNeuronCount; endNeuronIndex++) {
            checkWeightGradients[offsetEndLayerIndex][endNeuronIndex] = malloc(sizeof(double) * startNeuronCount);
            checkBiasGradients[offsetEndLayerIndex][endNeuronIndex] = 0.0;

            for (int startNeuronIndex = 0; startNeuronIndex < startNeuronCount; startNeuronIndex++) {
                checkWeightGradients[offsetEndLayerIndex][endNeuronIndex][startNeuronIndex] = 0;
            }
        }
    }
}

/**
 * Updates the check weight and bias gradients (computed numerically as opposed to through back propagation).
 *
 * @param model
 * @param input
 * @param targetOutput
 * @param checkWeightGradients
 * @param checkBiasGradients
 */
void updateCheckParameterGradients(struct Model* model, double input[], const double targetOutput[],
                                   double** checkWeightGradient[], double* checkBiasGradients[]) {
    static float epsilon = 1e-6;

    double preChangeTotalCost;
    double postChangeTotalCost;
    double costDifference;

    for (int endLayerIndex = 1; endLayerIndex < NUMBER_OF_LAYERS; endLayerIndex++) {
        int offsetEndLayerIndex = offsetLayer(endLayerIndex);
        int startLayerIndex = endLayerIndex - 1;

        int endNeuronCount = model->neuronsPerLayer[endLayerIndex];
        int startNeuronCount = model->neuronsPerLayer[startLayerIndex];

        for (int endNeuronIndex = 0; endNeuronIndex < endNeuronCount; endNeuronIndex++) {
            propagateInputForward(model, input);
            preChangeTotalCost = getTotalCost(model, targetOutput);

            model->biases[offsetEndLayerIndex][endNeuronIndex] += epsilon;

            propagateInputForward(model, input);
            postChangeTotalCost = getTotalCost(model, targetOutput);

            costDifference = postChangeTotalCost - preChangeTotalCost;
            double checkBiasDeltaInfluence = costDifference / epsilon;

            checkBiasDeltaInfluence *= model->learningRate;

            checkBiasGradients[offsetEndLayerIndex][endNeuronIndex] += checkBiasDeltaInfluence;

            model->biases[offsetEndLayerIndex][endNeuronIndex] -= epsilon;

            for (int startNeuronIndex = 0; startNeuronIndex < startNeuronCount; startNeuronIndex++) {
                propagateInputForward(model, input);
                preChangeTotalCost = getTotalCost(model, targetOutput);

                model->weights[offsetEndLayerIndex][endNeuronIndex][startNeuronIndex] += epsilon;

                propagateInputForward(model, input);
                postChangeTotalCost = getTotalCost(model, targetOutput);

                // Undo change
                model->weights[offsetEndLayerIndex][endNeuronIndex][startNeuronIndex] -= epsilon;

                costDifference = postChangeTotalCost - preChangeTotalCost;
                double checkWeightDeltaInfluence = costDifference / epsilon;

                checkWeightDeltaInfluence *= model->learningRate;

                checkWeightGradient[offsetEndLayerIndex][endNeuronIndex][startNeuronIndex] += checkWeightDeltaInfluence;
            }
        }
    }
}

void printCheckParamterGradients(struct Model *model, double** checkWeightGradients[], double* checkBiasGradients[]) {
    for (int endLayerIndex = 1; endLayerIndex < NUMBER_OF_LAYERS; endLayerIndex++) {
        int offsetEndLayerIndex = offsetLayer(endLayerIndex);
        int startLayerIndex = endLayerIndex - 1;

        int endNeuronCount = model->neuronsPerLayer[endLayerIndex];
        int startNeuronCount = model->neuronsPerLayer[startLayerIndex];

        for (int endNeuronIndex = 0; endNeuronIndex < endNeuronCount; endNeuronIndex++) {
            double checkBiasDelta = checkBiasGradients[offsetEndLayerIndex][endNeuronIndex];

            printf("Check Δ Bias[%i][%i] = %lf\n", endLayerIndex, endNeuronIndex, checkBiasDelta);

            for (int startNeuronIndex = 0; startNeuronIndex < startNeuronCount; startNeuronIndex++) {
                double checkWeightDelta = checkWeightGradients[offsetEndLayerIndex][endNeuronIndex][startNeuronIndex];

                printf("Check Δ Weight[%i][%i][%i] = %lf\n", endLayerIndex, endNeuronIndex, startNeuronIndex, checkWeightDelta);
            }
        }
    }
}
#endif

/**
 * @param model The model which the parameter gradients will be based on.
 * @param layerIndex The layer index whose weight deltas are being calculated.
 * @param baseDelta The base delta, equal to change in the cost function over change in
 * the weighted sum of the neuron value.
 * @param weightGradients The weight gradient to fill.
 * @param biasGradients The bias gradient to fill.
 */
void updateParameterGradients(struct Model *model, const double* targetOutput, double** weightGradients[],
                              double* biasGradients[]) {
    int outputNeuronCount = model->neuronsPerLayer[OUTPUT_LAYER];

    // Entry indexed by [layerIndex][neuronIndex] gives
    // Δ C / Δ Z[layerIndex, neuronIndex]
    double* errors[NUMBER_OF_LAYERS];

    errors[OUTPUT_LAYER] = malloc(sizeof(double) * outputNeuronCount);

    // Fill errors of output layers
    for (int outputNeuronIndex = 0; outputNeuronIndex < outputNeuronCount; outputNeuronIndex++) {
        double outputNeuronValue = model->values[OUTPUT_LAYER][outputNeuronIndex];
        double targetOutputNeuronValue = targetOutput[outputNeuronIndex];

        // Δ C_outputNeuronIndex / Δ A[OUTPUT_LAYER][outputNeuronIndex]
        double firstErrorComponent = model->getCostDerivative(outputNeuronValue, targetOutputNeuronValue);
        // Δ A[OUTPUT_LAYER][outputNeuronIndex] / Δ Z[OUTPUT_LAYER][outputNeuronIndex]
        double secondErrorComponent = model->getActivation(outputNeuronValue);
        // Δ C_outputNeuronIndex / Δ Z[OUTPUT_LAYER][outputNeuronIndex]
        double error = firstErrorComponent * secondErrorComponent;

        errors[OUTPUT_LAYER][outputNeuronIndex] = error;
    }

    // Fill errors of non-output layers
    for (int endLayerIndex = OUTPUT_LAYER; endLayerIndex > INPUT_LAYER; endLayerIndex--) {
        int startLayerIndex = endLayerIndex - 1;
        int offsetEndLayerIndex = offsetLayer(endLayerIndex);

        int startNeuronsCount = model->neuronsPerLayer[startLayerIndex];
        int endNeuronsCount = model->neuronsPerLayer[endLayerIndex];

        errors[startLayerIndex] = malloc(sizeof(double) * startNeuronsCount);

        for (int startNeuronIndex = 0; startNeuronIndex < startNeuronsCount; startNeuronIndex++) {
            double error = 0.0;

            for (int endNeuronIndex = 0; endNeuronIndex < endNeuronsCount; endNeuronIndex++) {
                double nextError = errors[endLayerIndex][endNeuronIndex];
                double nextWeight = model->weights[offsetEndLayerIndex][endNeuronIndex][startNeuronIndex];

                double activationValue = model->values[startLayerIndex][startNeuronIndex];
                double activationValueDelta = model->getActivationChange(activationValue);

                double errorInfluence = nextWeight * nextError * activationValueDelta;
                error += errorInfluence;
            }

            // Take average of errors, not sum
            error /= endNeuronsCount;

            errors[startLayerIndex][startNeuronIndex] = error;
        }
    }

    // Update weights and biases of all layers based on errors
    for (int endLayerIndex = OUTPUT_LAYER; endLayerIndex > INPUT_LAYER; endLayerIndex--) {
        int offsetEndLaterIndex = offsetLayer(endLayerIndex);
        int startLayerIndex = endLayerIndex - 1;

        int endNeuronCount = model->neuronsPerLayer[endLayerIndex];
        int startNeuronCount = model->neuronsPerLayer[startLayerIndex];

        for (int endNeuronIndex = 0; endNeuronIndex < endNeuronCount; endNeuronIndex++) {
            for (int startNeuronIndex = 0; startNeuronIndex < startNeuronCount; startNeuronIndex++) {
                double errorOfEndNeuronOfWeight = errors[endLayerIndex][endNeuronIndex];

                double valueOfStartNeuron = model->values[startLayerIndex][startNeuronIndex];

                double biasGradientInfluence = errorOfEndNeuronOfWeight;
                double weightGradientInfluence = errorOfEndNeuronOfWeight * valueOfStartNeuron;

                biasGradientInfluence *= model->learningRate;
                weightGradientInfluence *= model->learningRate;

                weightGradients[offsetEndLaterIndex][endNeuronIndex][startNeuronIndex] += weightGradientInfluence;
                biasGradients[offsetEndLaterIndex][endNeuronIndex] += biasGradientInfluence;
            }
        }
    }
}

/**
 * Updates the weight and bias values within {@code model}, given the gradients of the cost function
 * with respect to the weights and biases.
 *
 * @param model
 * @param weightGradients
 * @param biasGradients
 */
void updateParameterValues(struct Model *model, double** weightGradients[], double* biasGradients[]) {
    for (int endLayerIndex = 1; endLayerIndex < NUMBER_OF_LAYERS; endLayerIndex++) {
        int offsetEndLayerIndex = offsetLayer(endLayerIndex);

        int endNeuronCount = model->neuronsPerLayer[endLayerIndex];
        int startNeuronCount = model->neuronsPerLayer[endLayerIndex - 1];

        for (int endNeuronIndex = 0; endNeuronIndex < endNeuronCount; endNeuronIndex++) {
            double biasDelta = biasGradients[offsetEndLayerIndex][endNeuronIndex];

            // update bias
            model->biases[offsetEndLayerIndex][endNeuronIndex] -= biasDelta;

#if PRINT_BIAS_UPDATE
            printf("Δ Bias[%i][%i] = %lf\n", endLayerIndex, endNeuronIndex, -biasDelta);
            printf("Bias[%i][%i] = %lf\n", endLayerIndex, endNeuronIndex, model->biases[offsetEndLayerIndex][endNeuronIndex]);
#endif

            for (int startNeuronIndex = 0; startNeuronIndex < startNeuronCount; startNeuronIndex++) {
                double weightDelta = weightGradients[offsetEndLayerIndex][endNeuronIndex][startNeuronIndex];

                // update weight
                model->weights[offsetEndLayerIndex][endNeuronIndex][startNeuronIndex] -= weightDelta;

#if PRINT_WEIGHT_UPDATE
                printf("Δ Weight[%i][%i][%i] = %lf\n", endLayerIndex, endNeuronIndex, startNeuronIndex, weightDelta);
                printf("Weight[%i][%i][%i] = %lf\n", endLayerIndex, endNeuronIndex, startNeuronIndex, model->weights[offsetEndLayerIndex][endNeuronIndex][startNeuronIndex]);
#endif
            }
        }
    }
}

static int epochIndex = 0;

/**
 * Allocates memory for the weight and bias gradients.
 *
 * @param model
 * @param weightGradients
 * @param biasGradients
 */
void initParameterGradients(struct Model* model, double** weightGradients[], double** biasGradients) {
    for (int layerIndex = 1; layerIndex < NUMBER_OF_LAYERS; layerIndex++) {
        int offsetLayerIndex = offsetLayer(layerIndex);

        int endNeuronCount = model->neuronsPerLayer[layerIndex];
        int startNeuronCount = model->neuronsPerLayer[layerIndex - 1];

        biasGradients[offsetLayerIndex] = malloc(sizeof(double) * endNeuronCount);
        weightGradients[offsetLayerIndex] = malloc(sizeof(double*) * endNeuronCount);

        for (int endNeuronIndex = 0; endNeuronIndex < endNeuronCount; endNeuronIndex++) {
            biasGradients[offsetLayerIndex][endNeuronIndex] = 0.0; // 1 1
            weightGradients[offsetLayerIndex][endNeuronIndex] = malloc(sizeof(double) * startNeuronCount);

            for (int startNeuronIndex = 0; startNeuronIndex < startNeuronCount; startNeuronIndex++)
                weightGradients[offsetLayerIndex][endNeuronIndex][startNeuronIndex] = 0.0;
        }
    }
}

/**
 * Feeds the input values of the entry into the input array given.
 *
 * @param input
 * @param entry
 * @param inputColumnIndices
 * @param inputColumnIndicesCount
 */
void initInput(double input[], const double entry[], const int inputColumnIndices[], int inputColumnIndicesCount) {
    for (int inputColumnIndex = 0; inputColumnIndex < inputColumnIndicesCount; inputColumnIndex++) {
        int inputColumn = inputColumnIndices[inputColumnIndex];
        input[inputColumnIndex] = entry[inputColumn];
    }
}

/**
 * Feeds the target output values of entry given into the target output array given.
 *
 * @param targetOutput
 * @param entry
 * @param outputColumnIndices
 * @param outputColumnIndicesCount
 */
void initTargetOutput(double targetOutput[], const double entry[], const int outputColumnIndices[], int outputColumnIndicesCount) {
    printf("Entry Input %lf Entry Output %lf\n", entry[0], entry[1]);

    for (int outputColumnIndex = 0; outputColumnIndex < outputColumnIndicesCount; outputColumnIndex++) {
        int outputColumn = outputColumnIndices[outputColumnIndex];
        targetOutput[outputColumnIndex] = entry[outputColumn];
    }
}

/**
 * Tests how well {@code model} fits {@code data}, placing the results into {@code predictedOutputs} and {@costs}.
 *
 * @param model
 * @param data
 * @param inputColumnIndices
 * @param outputColumnIndices
 * @param predictedOutputs
 * @param costs
 */
void test(struct Model* model, struct Data* data, int inputColumnIndices[], int outputColumnIndices[], double* predictedOutputs[], double costs[]) {
    int inputNeuronCount = model->neuronsPerLayer[INPUT_LAYER];
    int outputNeuronCount = model->neuronsPerLayer[OUTPUT_LAYER];

    for (int entryIndex = 0; entryIndex < data->numberOfEntries; entryIndex++) {
        double *entry = data->elements[entryIndex];

        double input[inputNeuronCount];
        double targetOutput[outputNeuronCount];

        initInput(input, entry, inputColumnIndices, inputNeuronCount);
        initTargetOutput(targetOutput, entry, outputColumnIndices, outputNeuronCount);

        // forward propagation
        propagateInputForward(model, input);
        double cost = 0.0;

        for (int outputIndex = 0; outputIndex < outputNeuronCount; outputIndex++) {
            double value = model->values[OUTPUT_LAYER][outputIndex];
            predictedOutputs[entryIndex][outputIndex] = value;

            double targetValue = targetOutput[outputIndex];
            cost += model->getCost(value, targetValue);
        }

        // Take average cost
        cost /= outputNeuronCount;

        costs[entryIndex] = cost;
    }
}

/**
 * Trains the model on the given data.
 *
 * @param model
 * @param data Container for the data the model will be trained on.
 * @param inputColumnIndices The indices of the columns within {@code data} that are the input columns.
 * @param outputColumnIndices The indices of the columns within {@code data} that are the output columns.
 */
void train(struct Model* model, struct Data* data, int inputColumnIndices[], int outputColumnIndices[]) {
    // [offsetLayerIndex][endNeuronIndex in layerIndex][startNeuronIndex in layerIndex - 1]
    double** weightGradients[NUMBER_OF_LAYERS - 1];
    // [offsetLayerIndex][endNeuronIndex]
    double* biasGradients[NUMBER_OF_LAYERS - 1];

    // Allocate the storage for the weight and bias deltas, in addition
    // to initializing them all weight and bias deltas with values of 0
    initParameterGradients(model, weightGradients, biasGradients);

#if GRADIENT_CHECKING
    // indexed same way as weightGradients and biasGradients
    double** checkWeightGradients[NUMBER_OF_LAYERS - 1];
    double* checkBiasGradients[NUMBER_OF_LAYERS - 1];

    initCheckParameterGradients(model, checkWeightGradients, checkBiasGradients);
#endif

    int inputNeuronCount = model->neuronsPerLayer[INPUT_LAYER];
    int outputNeuronCount = model->neuronsPerLayer[OUTPUT_LAYER];
    epochIndex++;

    // Feed each input into model
    for (int entryIndex = 0; entryIndex < data->numberOfEntries; entryIndex++) {
        double* entry = data->elements[entryIndex];

        double input[inputNeuronCount];
        double targetOutput[outputNeuronCount];

        // Feed values of entry into input and targetOutput given indices of input and output columns
        initInput(input, entry, inputColumnIndices, inputNeuronCount);
        initTargetOutput(targetOutput, entry, outputColumnIndices, outputNeuronCount);

        // forward propagation
        propagateInputForward(model, input);

#if PRINT_EPOCH_UPDATE
        double cost = 0.0;

        for (int outputIndex = 0; outputIndex < outputNeuronCount; outputIndex++) {
            double value = model->values[OUTPUT_LAYER][outputIndex];
            double targetValue = targetOutput[outputIndex];
            cost += model->getCost(value, targetValue);
        }

        printf("Epoch %i, Entry %i, Total Cost %lf, \t\tCost %lf\n", epochIndex, entryIndex, cost, cost / outputNeuronCount);
#endif

        // update weight and bias gradients based on this entry, part of the batch
        updateParameterGradients(model, targetOutput, weightGradients, biasGradients);

#if GRADIENT_CHECKING
        updateCheckParameterGradients(model, input, targetOutput, checkWeightGradients, checkBiasGradients);
#endif
    }

    // now that
    updateParameterValues(model, checkWeightGradients, checkBiasGradients);

#if GRADIENT_CHECKING
    printCheckParamterGradients(model, checkWeightGradients, checkBiasGradients);
#endif

    // free the memory taken by weight and bias gradients
    for (int layerIndex = 1; layerIndex < NUMBER_OF_LAYERS; layerIndex++) {
        int offsetLayerIndex = offsetLayer(layerIndex);

        int neuronCount = model->neuronsPerLayer[layerIndex];

        free(biasGradients[offsetLayerIndex]);

        for (int neuronIndex = 0; neuronIndex < neuronCount; neuronIndex++)
            free(weightGradients[offsetLayerIndex][neuronIndex]);
    }
}

/**
 * Allocates the memory for the parameters (weights and biases) of the model, in addition to initializing
 * them to their default values.
 *
 * @param model
 */
void initParameters(struct Model* model) {
    // initialize weights with arbitrary
    for (int endLayerIndex = 1; endLayerIndex < NUMBER_OF_LAYERS; endLayerIndex++) {
        int offsetEndLayerIndex = offsetLayer(endLayerIndex);

        int endNeuronCount = model->neuronsPerLayer[endLayerIndex];
        int startNeuronCount = model->neuronsPerLayer[endLayerIndex - 1];

        model->weights[offsetEndLayerIndex] = malloc(sizeof(double*) * endNeuronCount);

        for (int endNeuronIndex = 0; endNeuronIndex < endNeuronCount; endNeuronIndex++) {
            model->weights[offsetEndLayerIndex][endNeuronIndex] = malloc(sizeof(double) * startNeuronCount);
            model->biases[offsetEndLayerIndex] = malloc(sizeof(double) * endNeuronCount);

            for (int startNeuronIndex = 0; startNeuronIndex < startNeuronCount; startNeuronIndex++) {
                model->weights[offsetEndLayerIndex][endNeuronIndex][startNeuronIndex] = model->getInitialWeightValue(startNeuronCount, endNeuronCount);
                model->biases[offsetEndLayerIndex][endNeuronIndex] = model->getInitialBiasValue(startNeuronCount, endNeuronCount);
            }
        }
    }
}

/**
 * Allocayes the memory for the values of the model.
 *
 * @param model
 */
void initValues(struct Model* model) {
    for (int layerIndex = 0; layerIndex < NUMBER_OF_LAYERS; layerIndex++) {
        int neuronsInLayer = model->neuronsPerLayer[layerIndex];
        model->values[layerIndex] = malloc(sizeof(double) * neuronsInLayer);
    }
}

I won't show data.c/h since it is irrelevant to the network itself (just loads data from CSV). Trust that initInput and initOutput, given an entry from this data, population a conventional input and output array as one would expect.

main.c

#include <stdio.h>
#include <stdlib.h>
#include <zconf.h>
#include "model.h"
#include "functions.h"
#include "data.h"

#define EPOCH_COUNT 100
#define NUMBER_OF_COLUMNS 3
#define TRAIN_ENTRIES_SIZE 4
#define TEST_ENTRIES_SIZE 4

#define PRINT_TEST_RESULTS true

int main() {
    struct Model model = {
            .neuronsPerLayer = {2, 5, 1},
            .learningRate = 0.2,

            // Default values
            .getActivation = getDefaultActivation,
            .getActivationChange = getDefaultActivationDerivative,
            .getCost = getDefaultCost,
            .getCostDerivative = getDefaultCostDerivative,
            .activationFunctionDerivativeUsesOutput = true,
            .getInitialWeightValue = getDefaultInitialWeightValue,
            .getInitialBiasValue = getDefaultInitialBiasValue,
    };

    int numberOfInputs = model.neuronsPerLayer[INPUT_LAYER];
    int numberOfOutputs = model.neuronsPerLayer[OUTPUT_LAYER];

    // Change working directory so data can be referenced relative to parent data folder
    chdir("..");
    chdir("data");

    struct Data trainData;
    fill(&trainData, "xor/train.csv", NUMBER_OF_COLUMNS, TRAIN_ENTRIES_SIZE);

    struct Data testData;
    fill(&testData, "xor/test.csv", NUMBER_OF_COLUMNS, TEST_ENTRIES_SIZE);

    int inputColumnIndices[numberOfInputs];
    int outputColumnIndices[numberOfOutputs];

    inputColumnIndices[0] = 0;
    inputColumnIndices[1] = 1;
    outputColumnIndices[0] = 2;

    initValues(&model);
    initParameters(&model);

    for (int epochIndex = 0; epochIndex < EPOCH_COUNT; epochIndex++)
        train(&model, &trainData, inputColumnIndices, outputColumnIndices);

    // Testing
    double* predictedOutputs[TEST_ENTRIES_SIZE];
    for (int predictedOutputIndex = 0; predictedOutputIndex < TEST_ENTRIES_SIZE; predictedOutputIndex++)
        predictedOutputs[predictedOutputIndex] = malloc(sizeof(double) * numberOfOutputs);

    double costs[TEST_ENTRIES_SIZE];

    test(&model, &testData, inputColumnIndices, outputColumnIndices, predictedOutputs, costs);

    for (int entryIndex = 0; entryIndex < TEST_ENTRIES_SIZE; entryIndex++) {
        double* entry = testData.elements[entryIndex];

        double inputs[numberOfInputs];
        double targetOutputs[numberOfOutputs];

        initInput(inputs, entry, inputColumnIndices, numberOfInputs);
        initTargetOutput(targetOutputs, entry, outputColumnIndices, numberOfOutputs);

#if PRINT_TEST_RESULTS
        printf("Inputs =");

        for (int inputIndex = 0; inputIndex < numberOfInputs; inputIndex++) {
            double input = inputs[inputIndex];
            printf(" %lf", input);
        }

        printf(", Target Outputs =");

        for (int outputIndex = 0; outputIndex < numberOfOutputs; outputIndex++) {
            double targetOutput = targetOutputs[outputIndex];
            printf(" %lf", targetOutput);
        }

        printf(", Predicted Outputs =");

        for (int outputIndex = 0; outputIndex < numberOfOutputs; outputIndex++) {
            double predictedOutput = predictedOutputs[entryIndex][outputIndex];
            printf(" %lf", predictedOutput);
        }

        printf(".\n");
#endif
    }

    exit(0);
}
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.