I seem to have taught myself the basics of c++ living off grig in the woods of Alaska. Most of this has been done on my android phone using cpp droid. This is the first time anyone has seen any of my code and I am curious what you think of it. There no error checking and the style may look a little strange on a normal computer. I do have a laptop, but limited power and a dead battery it makes it hard to use. I use Linux Mint 18 on it, I am not impressed with Windows or Mac.
This is my attempt at a feedforward backprop neural network with little knowledge of calculus. I am using only std libraries so it should be platform compatible.
After several attempts trying to follow Pdfs that I could never get to work I created my own. Some where several thousand lines of code. I'm hoping I created a program that runs as fast as possible. I am passing almost no parameters in my function and use the variables directly instead of methods. I think that using the Layers as a derived class of Neuron does not break encapsulation?
I had to comment out the protected: line in Neuron because it was giving me an error.
Do I have to derive Layer by saying :protected instead of :public?
My output layer only holds one neuron so I have not created a vector for it.
I would have used the hand written number detection to test but I could not figure out how to read the file.
There is something about flipping the bits in the header. I do know how to use fstream but it does not work with cpp droid.
I came up with the idea of teaching it to add two numbers but could not figure out how to add whole numbers so decimals it was.
I have yet for it to give a correct answer. It seems to know that .1 +.1 is less than .8+.1.
I understand that this is a naieve way to accomplish a neural network but I believe it would work with a little tweaking. I'll have to take my windows,YUK, laptop out and put a copiler on it to improve this program more.
My goal was to create a net for NLP. I found a list of 366,00 words and put them in a map numbered 1 through 366,000. as the key. If I understand neural networks that would be too many inputs for a net. How many would be too many. I tried to use n->weights[c] but it was giving incorrect values so I was forced to use layers[r][c].weights[w] I believe I am not creating my neurons right.
I am curious to see what you think about my code. Thank you in advance for you time and input. I may take a few days for me to check back in.
/*
This creates an implicitly connected feed forward
backward propagated deep neural network
this program has two inputs and one output
It is attempting to learn how to add two decimals together
*/
#include<iostream>
#include<vector>
#include<cmath>
#include<random>
#include <cassert>
using namespace std;
//Neurons hold the values
class Neuron {
public:
Neuron();
virtual ~Neuron() {}
void setValue(double val) {
_value =val;
}
void setTestValue(double val) {
_testValue = val;
}
//protected:
double derivative =0;
double _gradient =0;
double _delta =0;
double _testValue =0;
double _value =0;
double _bias =0;
vector<double> _weight;
};
Neuron::Neuron() {
//cout << "Making a neuron!" << endl;
}
//The layer class manipulates the values of neurons
class Layer :public Neuron {
public:
Layer() {}
Layer( vector<double>entry) {}
Layer(vector<int> topology, vector<double> input, vector<double> test);
~Layer() {}
void createNet();
void train();
void feed();
void predict(vector<double> entry);
void backward();
void printOutput();
double randNum();
double sigmoid(double x) {
return x/(1+abs(x));//Is this right?
}
double getError();
private:
double _momentum=.05;
double _mFactor =.05;
double _learn =.5;
double netError =0;
double _error =0;
double _rms =0;
double _answer =0;
// Neuron * n = new Neuron;
vector<Neuron> layer;
vector<vector<Neuron> > layers;
vector<int> topology;
};
Layer::Layer(vector<int> topology, vector<double> input, vector<double> test) {
this->topology=topology;
//r stands for row
for(unsigned r=0; r<topology.size(); r++) {
if(r==0) {
for(unsigned c=0; c< input.size(); c++) {// c stands for collum
Neuron i;
i.setValue(input.at(c));
layer.push_back(i);
}
} else if(r==topology.size()-1) {//output layer
for(int c=0; c<topology[r]; c++) {
Neuron o;
o._bias = 1;//= randNum();//would work either way to set neuron bias i think
//w stands for weight
for(auto w=0; w < topology[r-1]; w++) {//took away i-1
double weight = randNum();
o._weight.push_back(weight);
}
o.setTestValue(test.at(c));
layer.push_back(o);
o._weight.clear();
}
} else {//This creates the hidden layers
for (int c =0; c<topology[r]; c++) {
Neuron h;
h._bias = 1;
for(int w=0; w<topology[ r+1]; w++) {
double weight=randNum();
h._weight.push_back(weight);
}
layer.push_back(h);
h._weight.clear();
}
}
layers.push_back(layer);
layer.clear();
}
}
void Layer::createNet() {
vector<int> topology;
vector<double> input;
vector<double> test;
topology.push_back(2);//input layer
//There is no limit on the number of hidden layers
topology.push_back(10);//hidden layer
topology.push_back(10);//hidden layer
topology.push_back(1);//output layer
input.push_back(0.09);
input.push_back(.08);
test.push_back(.17);
Layer layer(topology, input, test);
int laps=0;
double entry=0;
vector<double>entries;
layer.feed();
layer.train();
layer.predict(entries);
}
void Layer::train() {
int lap=0;
int totalTrain=1000;
double sum=0;
double temp=0;
double avg=0;
double total=0;
while(lap<totalTrain) {
lap++;
sum=0;
//cout<<"-----LAP----- "<<lap<<endl;
for(auto r=0; r<layers.size(); r++) {
if(r==0) {
sum=0;
for(int c=0; c<layers[r].size(); c++) {
temp=randNum();
sum+=temp;
layers[r][c].setValue(temp);
}
}
if(r==layers.size()-1)
layers[r][0].setTestValue(sigmoid(sum));
}
double epoch=0;
double count=10;
// actual train loop
while(epoch<count) {
epoch++;
if(getError()<.001) {
//cout<<"BREAK on epoch "<<epoch<<endl;
temp+=epoch;
break;
} else {
temp+=epoch;
}
feed();
backward();
//cout<<getError()<<endl;
}
//here I'm trying to come up with overall net error
avg+= temp;
temp=0;
avg/=count;
total+=avg;
// cout<<"average "<<avg<<endl;
avg=0;
}
total/=totalTrain;
// cout<<"average training total "<<total<<endl;
//total=0;
}
void Layer::predict(vector<double> entry) {
double answer=0;
string response="y";
while(response=="y") {
for(auto r=0; r<layers.size(); r++) {
if(r==0) {
answer=0;
for(int c=0; c<layers[r].size(); c++) {
double temp=0;
cout<<"Enter a decimal number like .1 or .4 the total needs to be less than 1.0 "<<endl;
cin>>temp;
answer+=temp;
layers[r][c].setValue(temp);
//cout<<"value "<<layers[r][c]._value<<endl;
}
layers[layers.size()-1][0].setTestValue(answer);
}
}
feed();
printOutput();
cout<<"Do you want to continue (y/n)"<<endl;
cin>>response;
}
}
void Layer::feed() {
// cout << "********** FORWARD **********" << endl;
double sum=0;
for(unsigned r=1; r<layers.size(); r++) {
if(r == layers.size()-1) { //This is the output layer
for(unsigned c=0; c<layers[r].size(); c++) {
double sum=0;
// cout<<"Feed neron sum "<<layers[r][c]._value<<endl;
for(unsigned w=0; w < layers[r][c]._weight.size(); w++) {//size of weights should match previous number of neurons in previous layer
double pValue = layers[r-1][w]._value;
double weight = layers[r][c]._weight[w];
sum +=pValue * weight;
}
sum+=layers[r][c]._bias;
layers[r][c].setValue(sigmoid(sum));
//cout<<"Neuron Value " << layers[r][c]._value<<endl;
}
} else { //1st hidden layer of however many are made
for(unsigned c=0; c<layers[r].size(); c++) {
sum=0;
for(unsigned w=0; w < layers[r][c]._weight.size(); w++) {//size of weights should match previous number of neurons in previous layer
double pValue = layers[r-1][w]._value;
double weight = layers[r][c]._weight[w];
sum +=pValue * weight;
}
sum+=layers[r][c]._bias;
layers[r][c].setValue(sigmoid(sum));
}
}
}
}
void Layer::backward() {
double derivative=0;
double sum=0;
// cout << "********** Calculating gradients **********" << endl;
//AKA calculating SLOPE?
for(unsigned r=layers.size()-1; r>0; r--) {
if(r == layers.size()-1) {//output layer
for(unsigned c=0; c<layers[r].size(); c++) {
double val = layers[r][c]._value;
double test = layers[r][c]._testValue;
derivative=(1-val)*val;
double gradient=layers[r][c]._gradient=derivative*(test-val);
}
} else { //hidden row
for(unsigned c=0; c<layers[r].size(); c++) {
double val = layers[r][c]._value;
//derivative=(1-val)*(1+val);//used on tanf
derivative=(1-val)*val;
sum=0;
for(unsigned g=0; g < layers[r+1].size(); g++) {//gradients and weights are same size
sum +=layers[r+1][g]._gradient * layers[r][c]._weight[g];
layers[r][c]._gradient=derivative*sum;
}
}
}
}
// cout << "********** Updating Weights**********<<endl;
for(unsigned r=layers.size()-1; r >0; r--) {
if(r==layers.size()-1) {
for(unsigned c=0; c<layers[r].size(); c++) {
for(unsigned w=0; w<layers[r][c]._weight.size(); w++) {
double weight=layers[r][c]._weight[w];
//double val=layers[r-1][w]._value;
double cVal=layers[r-1][c]._value;
double grad=layers[r][c]._gradient;
double delta=layers[r][c]._delta;
double bias=layers[r][c]._bias;
double oldDelta=delta;
delta=_learn*grad*cVal;
//cout<<"Output weight adjust delta "<<delta<<" row "<<r<<" col "<<c<<endl;
weight+=delta;
_mFactor = _momentum*oldDelta;
layers[r][c]._weight[w]+=_mFactor;
//----------------now adjust neuron bias-----------------
delta =_learn*grad;
//cout<<"delta "<<delta<<endl;
bias+=delta;
_mFactor=_momentum*oldDelta;
layers[r][c]._bias=bias+=_mFactor;
}
}
} else {//adjust hidden weights
for(unsigned c =0; c< layers[r].size(); c++) {
double val = 0;
double grad =layers[r+1][c]._gradient;
double delta=layers[r][c]._delta;
double bias=layers[r][c]._bias;
for(unsigned w=0; w< layers[r][c]._weight.size(); w++) {
double weight=layers[r][c]._weight[w];
val = layers[r-1][c]._testValue;
double oldDelta=delta;
//cout<<"hidden weight adjust delta "<<delta<<" row "<<r<<" col "<<c<<endl;
// cout<<" weight "<<w<<endl;
delta=_learn*grad*val;
weight+=delta;
layers[r][c]._weight[w]+=_momentum*oldDelta;
//now adjust neuron bias
delta =_learn*grad;
bias+=delta;
//_mFactor=_momentum*oldDelta;
layers[r][c]._bias=bias+=_momentum*oldDelta;
}
}
}
}
}
void Layer::printOutput() {
for(auto r=0; r<layers.size(); r++) {
for(auto c=0; c<layers[r].size(); c++) {
if(r==layers.size()-1) {
cout<<"NEURON value "<<layers[r][c]._value<<endl;
cout<<"NEURON test answer "<<layers[r][c]._testValue<<endl;
}
}
}
}
double Layer::randNum() {
random_device rd;
mt19937 gen(rd());
uniform_real_distribution<> urd(0,1);
return urd(gen);
}
double Layer::getError() {
for(unsigned r=layers.size()-1; r>layers.size()-2; r--) {
double error=0;
double size=layers[r].size();
for(unsigned c=0; c<layers[r].size(); c++) {
double val=layers[r][c]._value;
double test=layers[r][c]._testValue;
error+=test-val;
}
error*=error;
error=error/size;
error=sigmoid(error);
return error;
}
}
int main() {
Layer layer;
layer.createNet();
cout << "##########FINISHED!!!!##########" << endl;
return 0;
}
