```1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 ``` ```import math import random import string random.seed(0) # calculate a random number where: a <= rand < b def rand(a, b): return (b-a)*random.random() + a # Make a matrix (we could use NumPy to speed this up) def makeMatrix(I, J, fill=0.0): m = [] for i in range(I): m.append([fill]*J) return m # our sigmoid function, tanh is a little nicer than the standard 1/(1+e^-x) def sigmoid(x): return math.tanh(x) # derivative of our sigmoid function, in terms of the output (i.e. y) def dsigmoid(y): return 1.0 - y**2 class NN: def __init__(self, ni, nh, no): # number of input, hidden, and output nodes self.ni = ni + 1 # +1 for bias node self.nh = nh self.no = no # activations for nodes self.ai = [1.0]*self.ni self.ah = [1.0]*self.nh self.ao = [1.0]*self.no # create weights self.wi = makeMatrix(self.ni, self.nh) self.wo = makeMatrix(self.nh, self.no) # set them to random vaules for i in range(self.ni): for j in range(self.nh): self.wi[i][j] = rand(-0.2, 0.2) for j in range(self.nh): for k in range(self.no): self.wo[j][k] = rand(-2.0, 2.0) # last change in weights for momentum self.ci = makeMatrix(self.ni, self.nh) self.co = makeMatrix(self.nh, self.no) def update(self, inputs): if len(inputs) != self.ni-1: raise ValueError('wrong number of inputs') # input activations for i in range(self.ni-1): #self.ai[i] = sigmoid(inputs[i]) self.ai[i] = inputs[i] # hidden activations for j in range(self.nh): sum = 0.0 for i in range(self.ni): sum = sum + self.ai[i] * self.wi[i][j] self.ah[j] = sigmoid(sum) # output activations for k in range(self.no): sum = 0.0 for j in range(self.nh): sum = sum + self.ah[j] * self.wo[j][k] self.ao[k] = sigmoid(sum) return self.ao[:] def backPropagate(self, targets, N, M): if len(targets) != self.no: raise ValueError('wrong number of target values') # calculate error terms for output output_deltas = [0.0] * self.no for k in range(self.no): error = targets[k]-self.ao[k] output_deltas[k] = dsigmoid(self.ao[k]) * error # calculate error terms for hidden hidden_deltas = [0.0] * self.nh for j in range(self.nh): error = 0.0 for k in range(self.no): error = error + output_deltas[k]*self.wo[j][k] hidden_deltas[j] = dsigmoid(self.ah[j]) * error # update output weights for j in range(self.nh): for k in range(self.no): change = output_deltas[k]*self.ah[j] self.wo[j][k] = self.wo[j][k] + N*change + M*self.co[j][k] self.co[j][k] = change #print N*change, M*self.co[j][k] # update input weights for i in range(self.ni): for j in range(self.nh): change = hidden_deltas[j]*self.ai[i] self.wi[i][j] = self.wi[i][j] + N*change + M*self.ci[i][j] self.ci[i][j] = change # calculate error error = 0.0 for k in range(len(targets)): error = error + 0.5*(targets[k]-self.ao[k])**2 return error def test(self, patterns): for p in patterns: print(p[0], '->', self.update(p[0])) def weights(self): print('Input weights:') for i in range(self.ni): print(self.wi[i]) print() print('Output weights:') for j in range(self.nh): print(self.wo[j]) def train(self, patterns, iterations=1000, N=0.5, M=0.1): # N: learning rate # M: momentum factor for i in range(iterations): error = 0.0 for p in patterns: inputs = p[0] targets = p[1] self.update(inputs) error = error + self.backPropagate(targets, N, M) if i % 100 == 0: print('error %-.5f' % error) print "Training ended." def demo(): # Teach network y=x function pat = [ [[1], [1]], [[2], [2]], [[3], [3]], [[4], [4]] ] # create a network with two input, two hidden, and one output nodes n = NN(1, 2, 1) # train it with some patterns n.train(pat, 1000, 0.7, 0.9) # test it n.test(pat) if __name__ == '__main__': demo() ```
 ```1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 ``` ```error 8.69058 error 7.00066 error 7.00036 error 7.00025 error 7.00019 error 7.00015 error 7.00013 error 7.00011 error 7.00010 error 7.00009 Training ended. ([1], '->', [0.999975892868742]) ([2], '->', [0.99998621092847673]) ([3], '->', [0.99998729704559453]) ([4], '->', [0.99998744603979561]) ```