# Academic implementation of artificial neural network

With some free time, I decided to study artificial neural networks as an academic exercise (not homework). Over the course of my studies, I decided to write a Python application that would allow me to create and train arbitrary feed forward networks so I could see the concrete math. So far, the test harness only includes simple AND, OR, NOT, and (slightly less simple) XOR with perfect inputs.

Quite honestly, in 8 years of professional development, I've never had my code reviewed. What I am looking for in code review is commentary on:

• my use of Python language features (not the decision to use Python, mind)
• the data structures and logical flows - especially if I have a logical flaw somewhere
• general program design

I'd also gladly welcome expert advice on ANN in general.

perceptron.py

"""
Equations from http://ecee.colorado.edu/~ecen4831/lectures/NNet3.html

t[i] = desired output of unit i
a[i] = actual output of unit i
a[i] = f(u[i]) = 1 / (1 - exp(-u[i])) # or experiment with other sigmoid functions
u[i] = sum(j=0..numinputs, W[i][j]*a[j]) + theta[i]

delta[i] = t[i] - a[i]
E = sum(i, pow(delta, 2)) # only over output units because we don't know what the activation of hidden units should be

dW[i][j] proportional to -(derivative(E) / derivative(W[i][j]))
.   # apply chain rule
.   = -(derivative(E)/derivative(a[i]))(derivative(a[i])/derivative(W[i][j]))
.   = 2(t[i] - a[i])*a[i]*(1-a[i])*a[j]
.   (NOTE: df[i] / du[i] = f[i]*(1 - f[i]) = a[i]*(1 - a[i]))

if unit i is output:
.   change_rate[i] = (t[i] - a[i]) * a[i] * (1-a[i])
if unit i is hidden, we don't know the target activation, so we estimate it:
.   change_rate[i] = (sum(k, dr1[k] * W[k][i])) * a[i] * (1-a[i])
.   ASSUME: sum(k, dr1[k] * W[k][i]) is approximately (t[i] - a[i])
.   (The sum over k is the sum over the units that receive input from the ith unit.)

dW[i][j](n+1) = learning rate * change_rate[i] * a[j] + alpha*dW[i][j](n), j=0..numinputs
"""
import math

def i_report_pattern(activations, training_values, change_rates, weights, dweights):
"""
Empty prototype for reporting the state of a network following a training pattern.

activations:
.    A dictionary of node index and output value. This must include the raw
.    input nodes, and must have as many elements as nodes in the ANN.
change_rates:
.    A dictionary of node index and delta function value used to calculate weight
.    changes in training.
weights:
.    A dictionary of node index and array of weights with theta appended.
dweights:
.    A dictionary of node index and array of deltas to be applied to the
.    weights and theta.

Example from perceptron of 2 inputs, 1 hidden node, and 1 output node
(1st epoch, 1st pattern):
.  activations = { 0:0.00, 1:0.00, 2:0.50, 3:0.50 }
.  training_values = { 3:0.00 }
.  change_rates = { 2:0.0000, 3: -0.1250 }
.  weights = { 2:[0.0000,0.0000,0.0000], 3:[0.0000,0.0000,0.0000,0.0000] }
.  dweights = { 2:[0.0000,0.0000,0.0000], 3:[0.0000,0.0000,-0.0156,-0.0312] }
"""
pass

class TrainingStrategy(object):
PATTERN = "P"
EPOCH = "E"

class Node(object):
"""
Represents one neuron in a perceptron with memory of last input and all
weight changes applied during the life of this instance.
"""

def __init__(self, numinputs):
self.numinputs = numinputs
self.weights = *numinputs
self.theta = 0
self.last_input = None
# Remember all dW & dTheta values until they are applied to the weights and theta.
self.temp_delta_memory = []
# Remember all dW & dTheta values for life. dTheta is appended to dW here for simplicity.
self.perm_delta_memory = [(*(numinputs),0)]

def __str__(self): return '{} ; {} ; {}'.format(str(self.weights), self.theta, str(self.temp_delta_memory))

def accept_input(self, input_vector, transform_function):
"""
Apply the transform function to the results of sum(V[i] * W[i]) + Theta,
i indexes each element in the input_vector.

input_vector: an enumerable of scalar values, the length of which must
.             match the number inputs specified during initialization.
transform_function: a scalar function with one scalar input. Examples
.                   include the step function (1 if x >= 0 else 0) and
.                   the sigmoid function 1 / (1 - exp(-x)).
"""
def sum_function(V, W, Theta):
return sum(v * W[vi] for vi,v in enumerate(V)) + Theta

self.last_input = input_vector
return transform_function(sum_function(input_vector, self.weights, self.theta))

def remember_deltas(self, delta, learning_epsilon, learning_acceleration):
"""
Calculate dW[i] and dTheta based on the last input value, the training delta,
learning rate, learning acceleration, and pervious dW[i] and dTheta values.
These values are temporarily stored instead of immediately applied so they can
be applied following an epoch, if desired. These values are also permanently
stored for the life of this node.

delta: scalar value representing the difference between expected output and actual.
learning_epsilon: multiplier to control the magnitude of weight change
learning_acceleration: provides momentum in the learning process and is multiplier
.                      on previous values of dW[i] and dTheta
"""

# add the previous deltas times a learning acceleration factor (learning_acceleration) both to
# help prevent radical weight changes and to apply some momentum to learning.
p_dweights, p_dtheta = self.perm_delta_memory[-1]
magnitude = delta * learning_epsilon

dW = [magnitude * self.last_input[vi] + learning_acceleration * p_dweights[vi] for vi in range(self.numinputs)]
dTheta = magnitude + learning_acceleration * p_dtheta

self.temp_delta_memory.append((dW, dTheta)) # remember until deltas are applied to weights
self.perm_delta_memory.append((dW, dTheta)) # remember for life.

def apply_deltas(self):
"""
Add all previously calculated dW[i] and dTheta values to W[i] and Theta, then
forget the dWs and dThetas.
"""
for dW, dTheta in self.temp_delta_memory:
self.weights = [w + dW[wi] for wi,w in enumerate(self.weights)]
self.theta += dTheta
self.temp_delta_memory = []

class FFPerceptron(object):
"""
A feed-forward artificial neural network with configurable neural pathways.
"""
def __init__(self, transform_function, links):
"""
transform_function:
.    Any scalar function with one input, which will be applied to the output of every
.      node. The intent is to support arbitrarily step functions, sigmoid functions, and
.      unmodified output.
links:
.    A collection of pairs of input vectors and output vectors.
.    Inputs will be sent in specified order to each output. Order of processing within
.      a particular  output vector is not guaranteed.
.    LIMITATIONS:
.       Output nodes should only appear once in any given output vector.
.       An output must have a consistent number of inputs if it is to exist in
.         multiple output vectors.
.       A node may only receive one input vector, because it will only be
.         processed once per system input.
"""
from collections import defaultdict

sender_node_ixs = set(ni for iv,ov in links for ni in iv) # all nodes that send input to another node.
receiver_node_ixs = set(ni for iv,ov in links for ni in ov) # all nodes that receive input from another node.
nodes = defaultdict(lambda:None) # all nodes with calculation function
output_paths = defaultdict(list) # tracks which inputs go to which nodes

# generate node instances and build output paths.
for input_vector,output_vector in links:
for node_index in output_vector:
if node_index not in nodes: # In case a node shows up in multiple output vectors.
nodes[node_index] = Node(len(input_vector))
for node_position,node_index in enumerate(input_vector):
output_paths[node_index].extend([{'atposition':node_position, 'tonode':output_node_index} for output_node_index in output_vector])

self.links = links
self.output_node_ixs = receiver_node_ixs - sender_node_ixs # nodes that output from the system.
self.input_node_ixs = sender_node_ixs - receiver_node_ixs # nodes that are input to the system.
self.nodes = nodes
self.output_paths = output_paths
self.transform_function = transform_function
self.activation_memory = [{i:0 for i in range(len(self.nodes))}]

def __str__(self): return '::'.join(str(n) for n in self.nodes)

def accept_input(self, input_vector):
"""
Processes the input vector through all nodes and returns the final activation
value(s). Remembers all activations for future training.

input_vector: an enumerable of scalar values, the length of which must
.             match the number input nodes specified in the links parameter
.             during initialization
"""
from collections import defaultdict

# remember all activations by node index.
activations = defaultdict(lambda:0)

# input values are easy.
for vi,v in enumerate(input_vector): activations[vi] = v

for linkin,linkoutlist in self.links:
# Sequentally calculate and remember the activations for all non-input nodes.
# Only process a node once.
for node_index in linkoutlist:
if node_index not in activations:
v = tuple(activations[i] for i in linkin)
node = self.nodes[node_index]
activations[node_index] = node.accept_input(v, self.transform_function)

self.activation_memory.append(activations)
return (activations[i] for i in self.output_node_ixs)

def train(self, reporter, tset, error_precision, learning_epsilon, learning_acceleration, max_iterations, training_strategy):
"""
Attempt to train the network according to scenarios specified in tset, iterating the scenarios until the
network is trained or max_iterations is reached. Report the states of all nodes after each pattern is run.
reporter: a class containing a concrete implementation of the method i_report_pattern (top of file).
tset: an array of tuples containing an input vector and a dictionary of expected outputs. The collection
.     represents one training epoch; each element one training pattern.
.     (e.g.): [((0,0),{2:0}), ((0,1),{2:1}), ((1,0),{2:1}), ((1,1),{2:0})]
error_precision: error epsilon is calculated 10^-error_precision. The network is trained when the
.     epoch error <= error epsilon.
.     (e.g.): error_precision = 3, error_epsilon = 0.001
learning_epsilon: (learning rate) the rate at which the network learns. Too low a value will cause the
.     network to learn too slowly and not train within the max_iterations. Too high a value could cause
.     the network to continuously overshoot the target and fail to learn.
learning_acceleration: similar to learning rate, this provides some momentum to learning as previous
.     deltas multiplied by this value are added to the deltas of the current training session. Too high
.     a value here could cause a numeric overflow, while too low can make learning too slow.
training_strategy: A value from TrainingStrategy - indicates when to modify the weights (after a pattern
.     is complete or after an epoch).
"""

def run_pattern(V, T):
"""
Execute one training pattern in the training set. Calculate and save learning values.
Report state of all nodes. Train if specified by training_strategy. Return error.

V: input vector
T: expected output(s) of system.
"""
from collections import defaultdict

self.accept_input(V)
activations = self.activation_memory[-1] # these are the outputs from this run.

delta_error = 0

# calculate change rate for output nodes
change_rate = defaultdict(lambda:0)
for node_index in self.output_node_ixs:
a = activations[node_index]
diff = T[node_index] - a
delta_error += diff
# Literature concerning single-layer perceptrons seem to only use the delta-rule (T[i] - a) in the
# training equation, where-as (T[i]-a) * a * (1 - a) comes up with multi-layer perceptrons. I
# observe that the latter works well with single-layers as well. However, using a step-function
# guarantees the latter is always 0. (e.g.): a * (1 - a) == 0 for a in [0,1].
change_rate[node_index] = diff if a in (0,1) else diff * a * (1 - a)
self.nodes[node_index].remember_deltas(learning_epsilon, change_rate[node_index], learning_acceleration)

# calculate change rate for hidden nodes (dr for output nodes required)
for node_index in sorted(self.output_paths.keys(), reverse=True):
if node_index not in self.nodes: continue ## Must be a raw input node.

path = self.output_paths[node_index]
a = activations[node_index]
change_rate[node_index] = sum(change_rate[step["tonode"]] * self.nodes[step["tonode"]].weights[step["atposition"]] for step in path) * a * (1 - a)
self.nodes[node_index].remember_deltas(learning_epsilon, change_rate[node_index], learning_acceleration)

reporter.report_pattern(activations, T, change_rate,
{ni:n.weights+[n.theta] for ni,n in self.nodes.items()},
# perm memory is a tuple of weights, theta
# index -1 points to most recent memory.
{ni:n.perm_delta_memory[-1]+[n.perm_delta_memory[-1]] for ni,n in self.nodes.items()})

if training_strategy == TrainingStrategy.PATTERN:
for node in self.nodes.values():
node.apply_deltas()

return (delta_error * delta_error)

EPSILON = math.pow(10, -error_precision)

epochs = []
epoch = 0
trained = False
while not trained and epoch < max_iterations:
error = 0
reporter.start_epoch()
for V, T in tset:
error += run_pattern(V, T)

reporter.end_epoch(error)

trained = round(math.fabs(error), error_precision) <= EPSILON
if training_strategy == TrainingStrategy.EPOCH and not trained:
for node in self.nodes.values():
node.apply_deltas()

epoch += 1

return trained


perceptron_test.py

import math
import itertools
import perceptron

class Reporter(object):
"""
The purpose of this class is to capture values from an ANN throughout the course
of training, and then display those values in a tabular report. The display is
dynamically generated based on the network's configuration, which is specified
explicitly during initialization.

At least one call to start_epoch is required before reporting any data can be
reported. The assumption is that each epoch will be marked with matching calls
to start_epoch and end_epoch, so that report data can be grouped by epoch and
the Error Function per epoch can be reported.

Assume that the activation threshold theta is part of the sum function and
behaves as a weight with an internal input value (typically 1).

Terms:
epoch - one training cycle
activation - the output of a particular node (note raw input nodes have the
.            same output as input)
pattern - one set of inputs and expected outputs. An epoch typically consists
.         of multiple patterns.

Fields displayed:
.  * Activations from all nodes including raw input nodes.
.  * Expected activations from all output nodes (training values).
.  * Activation deltas for output nodes.
.  * The weights for all nodes in the network when the activations were calculated.
.  * The deltas to be applied to each weight during the training cycle.

(e.g.):
.  Given a network with 2 raw inputs and 1 output node (as in a simple AND gate),
.  the report would contain this header (the line is broken here, but not in the
.  report):
.      A0    |   A1    |   A2    |   T2    |   Dr2   |   W20   |   W21   |
.      Th2   |  dW20   |  dW21   |  dTh2   |
.  * Note A0, A1 are the raw inputs. Observe that fields match along the node
.  * indexes (Dr2 is derived from T2 and A2 and applied to the change values
.  * for W2* and Th2).
.
.  Given a network with 2 raw inputs, 1 hidden node, and 1 output node (as in an XOR
.  gate), the report would contain this header (the line is broken here, but not in
.  the report):
.      A0    |   A1    |   A2    |   A3    |   T3    |   Dr2   |   Dr4   |   W20   |
.      W21   |   Th2   |   W40   |   W41   |   W42   |   Th4   |  dW20   |  dW21   |
.      dTh2   |  dW40   |  dW41   |  dW42   |  dTh4   |
"""

HFMT = "{:^11s}|"
FMT = "{:^11,.3g}|"

def __init__(self, inputnodes, weightconfig, outputnodes):
"""
inputnodes:
.    Iterable of node indices for values input into the network. Used to calculate
.    the number of fields expected in all reported values. (e.g.): [0,1]
weightconfig:
.    Dictionary of nodes by index with number of weights as a value. Do not account
.    for the threshold theta in this number as it will be accounted for implicitly.
.    This configuration is used to calculate the number of fields expected in all
.    reported values.
.    (e.g.): {2:2, 3:2, 4:2} for a system with 2 inputs, 2 hidden nodes with 2
.                            weights each, and a single output node with 2 weights.
.                            In this case, the hidden nodes 2&3 are the only inputs
.                            into the output node, 4.
outputnodes:
.    Iterable of node indices. Used only to create
.    Training field headers (e.g.): [node_index,...]
"""
fields = ['A{}'.format(ni) for ni in range(len(inputnodes) + len(weightconfig))]
fields += ['T{}'.format(ni) for ni in outputnodes]
fields += ['CR{}'.format(ni) for ni,nw in weightconfig.items()]
fields += ['Th{}'.format(li) if nwi==nw else 'W{}{}'.format(li,nwi) for li,nw in weightconfig.items() for nwi in range(nw+1)]
fields += ['dTh{}'.format(li) if nwi==nw else 'dW{}{}'.format(li,nwi) for li,nw in weightconfig.items() for nwi in range(nw+1)]

self.epochs = []
self.fields = fields
self.numfields = len(self.fields)

def start_epoch(self):
"""
Mark the beginning of a training cycle. This is useful for grouping
purposes and for reporting the Error Function value for the cycle.
"""
self.epochs.append({"error":0,"patterns":[]})

def end_epoch(self, error):
"""
Mark the end of a training cycle.
"""
epoch = self.epochs[-1]
epoch["error"] = error

def report_pattern(self, activations, training_values, change_rates, weights, dweights):
"""
Concrete implementation of perceptron.i_report_pattern

NOTE: The length of each array in weights and dweights is one more than the
weightconfig specified in initialization. This extra element is, of course, the
activation threshold, theta.
"""

# flatten all fields into a 1-D array for ease of later printing.
pattern = [activations[k] for k in sorted(activations.keys())]
pattern += [training_values[k] for k in sorted(training_values.keys())]
pattern += [change_rates[k] for k in sorted(change_rates.keys())]
pattern += list(itertools.chain.from_iterable(weights.values()))
pattern += list(itertools.chain.from_iterable(dweights.values()))
self.epochs[-1]["patterns"].append(pattern)

def decipher_training_result(self, trained):
""" Convert tri-state value (boo nullable boolean) trained to single-character identifier. """
if trained is None: return "E"
return "T" if trained else "F"

def write_summary(self, scenario, trained, f):
"""
Display the results of a training session in a fixed-column format.

scenario: Tuple identifies name of scenario and all parameters:
.    (scenario name, transform function, training strategy, learning epsilon, learning acceleration)
trained: boolean value indicating whether the ANN was successfully trained.
f: an open file for text writing

Output fields:
.    Name - 17 chrs
.    Transform Function - 24 chrs
.    Training Strategy - 3 chrs (P - train after each pattern, E - train after each epoch)
.    Learning Epsilon - 7 chrs
.    Learning Acceleration - 7 chrs
.    Training Results - 3 chrs (E - error, F - failed, T - trained)
.    Number of epochs trained.
"""
name,tf,tp,le,la = scenario
f.write('{:17s}{:24s}{:3s}{:<7,.4g}{:<7,.4g}{:3s}{:<4,d}\n'.format(name, tf.__name__, tp, le, la, self.decipher_training_result(trained), len(self.epochs)))

def write_details(self, scenario, trained, f):
"""
Display the results of a training session with tabular detail.

scenario: Tuple identifies name of scenario and all parameters:
.    (scenario name, transform function, training strategy, learning epsilon, learning acceleration)
trained: boolean value indicating whether the ANN was successfully trained.
f: an open file for text writing
"""
def print_header():
f.write(''.join(self.HFMT.format(k) for k in (self.fields)))
f.write('\n')

def print_separator():
f.write(''.join(["-----------+" for i in range(self.numfields)]))
f.write('\n')

name,tf,tp,le,la = scenario
f.write('{}[TF:{}, T:{}, LE:{:g}, LA:{:g}] {} in {} epochs.\n'.format(name,
tf.__name__,
tp, le, la,
self.decipher_training_result(trained),
len(self.epochs)))
print_header()
print_separator()
for epoch in self.epochs:
# All fields were flattened into a 1-D array for ease here.
f.write('\n'.join(''.join(self.FMT.format(k) for k in pattern) for pattern in epoch["patterns"]))
f.write('Error: {:.9f}\n'.format(epoch["error"]))
print_separator()

def transform_step(x): return 1 if x >= 0 else 0
def transform_sigmoid_abs(x): return x / (1 + math.fabs(x))
def transform_sigmoid_exp(x): return 1 / (1 + math.exp(-x))

def truncate_file(fname):
with open(fname, "w+") as f:
pass

BITS = (0,1)
TSET = { # Training Sets
"OR":[((x,y),{2:int(x or y)}) for x in BITS for y in BITS],
"AND":[((x,y),{2:int(x and y)}) for x in BITS for y in BITS],
"NOT":[((x,),{1:int(not x)}) for x in BITS],
"XOR":[((x,y),{3:int(x^y)}) for x in BITS for y in BITS]
}
MAX_ITERATIONS = 256
ERROR_PRECISION = 3
PERMS = [(le, la, tf, ts)
for tf in [transform_step, transform_sigmoid_abs, transform_sigmoid_exp]
for ts in [perceptron.TrainingStrategy.PATTERN, perceptron.TrainingStrategy.EPOCH]
for le in [0.25, 0.5, 0.75]
for la in [0.75, 0.995, 1.0, 1.005, 1.1]]

SCENARIOS = [("OR",  TSET["OR"], [((0,1),(2,))], le, la, tf, ts) for le, la, tf, ts in PERMS] + [
("AND", TSET["AND"], [((0,1),(2,))], le, la, tf, ts) for le, la, tf, ts in PERMS] + [
("NOT", TSET["NOT"], [((0,),(1,))], le, la, tf, ts) for le, la, tf, ts in PERMS] + [
("XOR", TSET["XOR"], [((0,1),(2,)),((0,1,2),(3,))], le, la, tf, ts) for le, la, tf, ts in PERMS]

LOG_FILE = r"c:\temp\perceptron_error.log"
SUMMARY_FILE = r"c:\temp\perceptron_summary.txt"
DETAILS_FILE = r"c:\temp\perceptron_details.txt"

WRITE_DETAILS = True

truncate_file(LOG_FILE)
truncate_file(SUMMARY_FILE)
truncate_file(DETAILS_FILE)

for name, tset, pconfig, epsilon, acceleration, xform_function, training_strategy in SCENARIOS:
p = perceptron.FFPerceptron(xform_function, pconfig)
reporter = Reporter(p.input_node_ixs, {ni:len(n.weights) for ni,n in p.nodes.items()}, p.output_node_ixs)

try:
trained = p.train(reporter, tset, ERROR_PRECISION, epsilon, acceleration, MAX_ITERATIONS, training_strategy)
except Exception as err:
with open(LOG_FILE, "a+") as flog:
flog.write('{}\n{}\n'.format(str(p),repr(err)))
trained = None
#raise

scenario = (name, xform_function, training_strategy, epsilon, acceleration)
with open(SUMMARY_FILE, "a") as fsummary:
reporter.write_summary(scenario, trained, fsummary)
if WRITE_DETAILS:
with open(DETAILS_FILE, "a") as fdetails:
reporter.write_details(scenario, trained, fdetails)


## 1 Answer

Consider breaknig out the training logic from the network so there can be instances of non-trainable perceptrons. Presumably in this case there would be a mechanism for saving and loading weight data (by using pickle for example), which becomes easier without the pollution of the training data.

The "enum" TrainingStrategy is for controlling program flow. There should be a more organic way of doing this - "train_pattern(...)" and "train_epoch(...)".

The training sets include node index with the expected output. This prevents using the same training set on different configurations - or at least - severely complicates it. It would be more flexible for expected outputs to be a tuple that the training logic can interpret based on network configuration.

Here is example using a graph to implement node connections (doesn't include training logic):

class Node:
def __init__(self, node_index):
self.node_index = node_index
self.output_map = []

def map_output(self, node, position):
self.output_map.append((node,position))

def __repr__(self):
return '{}:{}'.format(self.node_index, repr(self.output_map))

class InputNode(Node):
def __init__(self, node_index):
Node.__init__(self, node_index)
self.input = 0

def evaluate(self):
a = self.input
for node,position in self.output_map:
node.input[position] = a

class Neuron(Node):
def __init__(self, node_index, num_inputs):
Node.__init__(self, node_index)
self.weights =  * (num_inputs + 1)
self.input =  * (num_inputs) + 
self.u = 0
self.a = 0

def evaluate(self):
def g(v, w):
return sum(v[i] * w[i] for i in range(len(v)))

def f(u):
return 0 if u < 0 else 1

self.u = g(self.input, self.weights)
self.a = f(self.u)

for node,position in self.output_map:
node.input[position] = self.a

def configure(config):
in_idx, map_output = config
nodes = [InputNode(idx) for idx in in_idx]
for node_index, mapping in map_output:
node = Neuron(node_index, len(mapping))
nodes.append(node)
for position, in_node_index in enumerate(mapping):
nodes[in_node_index].map_output(node, position)
return nodes

def test(nodes, tset):
for i,t in tset:
nodes.input = i
nodes.input = i

for n in nodes:
n.evaluate()

output_nodes = [node for node in nodes if len(node.output_map) == 0]
first = min(node.node_index for node in output_nodes)
for node in output_nodes:
node_index = node.node_index
tn = t[node_index - first]
print ('V{} = {}, A{} = {}, T{} = {}, D{} = {}'.format(node_index, node.input, node_index, node.a, node_index, tn, node_index, tn - node.a))

tmnodes = configure(((0,1,2,3),[(4,(0,1,2,3)),(5,(0,1,2,3)),(6,(4,5)),(7,(4,5)),(8,(4,5)),(9,(4,5))]))
tnodes = configure(((0,1),[(2,(0,1)),(3,(0,2,1))]))

mnodes = [InputNode(0), InputNode(1), Neuron(2, 2), Neuron(3, 3)]
mnodes.map_output(mnodes, 0)
mnodes.map_output(mnodes, 0)
mnodes.map_output(mnodes, 1)
mnodes.map_output(mnodes, 2)
mnodes.map_output(mnodes, 1)

print (tnodes)
print (mnodes)
print (tmnodes)

BITS = (0,1)
TSET_XOR = [((x,y),((x^y),)) for x in BITS for y in BITS]
TSET_ENCODER = [((w,x,y,z),(w,x,y,z)) for w in BITS for x in BITS for y in BITS for z in BITS]

test(tnodes, TSET_XOR)
test(mnodes, TSET_XOR)
test(tmnodes, TSET_ENCODER)