1
\$\begingroup\$

So after watching week 5 of the machine learning course on Coursera by Andrew Ng, I decided to write a simple neural net from scratch using Python. Here's my code:

import numpy as np
import csv
global e
global epsilon
global a
global lam
global itr
e = 2.718281828
epsilon = 0.12
a = 1
lam = 1
itr = 1000

# The sigmoid function(and its derivative)
def sigmoid(x, derivative=False):
    if derivative:
        return sigmoid(x) * (1 - sigmoid(x))
    return 1 / (1 + e**-x)

# The cost function
def J(X, theta1, theta2, y, lam, m):
    j = 0 
    for i in range(m):
        # The current case
        currX = X[i].reshape(X[i].shape[0], 1)
        z2 = theta1 @ currX
        a2 = sigmoid(z2)
        a2 = np.append([1], a2).reshape(a2.shape[0] + 1, 1)
        z3 = theta2 @ a2
        a3 = sigmoid(z3)
        j += sum(-y[i] * np.log(a3) - (1 - y[i]) * np.log(1 - a3)) / m + (lam / (2 * m)) * (sum(sum(theta1[:, 1:] ** 2)) + sum(sum(theta2[:, 1:] ** 2)))
    return j

# The gradients
def gradient(X, theta1, theta2, y, lam, m):
    theta1Grad = np.zeros(theta1.shape)
    theta2Grad = np.zeros(theta2.shape)
    Delta1 = np.zeros(theta1.shape)
    Delta2 = np.zeros(theta2.shape)
    for i in range(m):
        # The current case
        currX = X[i].reshape(X[i].shape[0], 1)
        z2 = theta1 @ currX
        a2 = sigmoid(z2)
        a2 = np.append([1], a2).reshape(a2.shape[0] + 1, 1)
        z3 = theta2 @ a2
        a3 = sigmoid(z3)
        delta3 = a3 - y[i]
        delta2 = theta2[:, 1:].T @ delta3 * sigmoid(z2, derivative=True)
        Delta1 += delta2 @ currX.reshape(1, -1)
        Delta2 += delta3 * a2.reshape(1, -1)
    theta1Grad = Delta1 / m
    theta2Grad = Delta2 / m
    theta1Grad[:, 1:] += (lam / m) * theta1[:, 1:]
    theta2Grad[:, 1:] += (lam / m) * theta2[:, 1:]
    thetaGrad = np.append(theta1Grad.reshape(theta1Grad.shape[0] * theta1Grad.shape[1], 1), theta2Grad.reshape(theta2Grad.shape[0] * theta2Grad.shape[1], 1))
    thetaGrad = thetaGrad.reshape(thetaGrad.shape[0], 1)
    return thetaGrad

# Gradient descent
def gradientDescent(X, theta1, theta2, y, lam, m):
    for i in range(itr):
        grad = gradient(X, theta1, theta2, y, lam, m)
        theta1Grad = grad[0:theta1.shape[0] * theta1.shape[1]].reshape(theta1.shape)
        theta2Grad = grad[theta1.shape[0] * theta1.shape[1]:].reshape(theta2.shape)
        theta1 = theta1 - a * theta1Grad
        theta2 = theta2 - a * theta2Grad
    return (theta1, theta2)

with open('data.csv', 'r') as f:
    data = csv.reader(f)
    d = []
    c = 0
    # Read the data
    for row in data:
        # Don't add the first line(it's our features' labels)
        if c == 0:
            c += 1
            continue
        curr_row = []
        k = 0
        for j in row:
            if j != '':
                if k == 1:
                    # Add a 1 between the y and x values(for the bias)
                    curr_row.append(1)
                curr_row.append(float(j))   
                k += 1
        d.append(curr_row)
    d = np.array(d)
    x = d[:, 1:]
    y = d[:, 0]
    # Split the data into training cases(80%) and test cases(20%)
    x_train = x[0:(d.shape[0]//5) * 4, :]
    y_train = y[0:(d.shape[0]//5) * 4]
    x_test = x[(d.shape[0]//5) * 4 : d.shape[0], :]
    y_test = y[(d.shape[0]//5) * 4 : d.shape[0]]
    # Initialize theta(s)
    theta1 = np.random.rand(5, x[0].shape[0]) * 2 * epsilon - epsilon
    theta2 = np.random.rand(1, 6) * 2 * epsilon - epsilon
    print(J(x_train, theta1, theta2, y_train, lam, x_train.shape[0]))
    theta1, theta2 = gradientDescent(x_train, theta1, theta2, y_train, lam, x_train.shape[0])

Please note a that my data only has 2 possible outputs so no need for one-vs-all classification.

Thanks in advance!

| improve this question | | | | |
\$\endgroup\$
  • \$\begingroup\$ What is this code supposed to do? what is its use? \$\endgroup\$ – user203258 Sep 22 '19 at 0:44
  • 1
    \$\begingroup\$ You don't need to declare global var in global scope. This only needs to be done inside a function that might need to modify a global variable \$\endgroup\$ – C.Nivs Sep 22 '19 at 2:29
  • \$\begingroup\$ @EmadBoctor This is a neural network(classifier) \$\endgroup\$ – Borna Ahmadzade Sep 22 '19 at 11:09

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.