handwrite/Network.py at master · mjayasur/handwrite · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
'''
A neural network.

:author: Michael Jayasuriya
'''
from random import random, shuffle
import numpy

class Network:
    '''
    Represents a layerSize network of Sigmoid Neurons.
    '''
    def __init__(self, layerSizes, learningRate = 3.0):
        '''
        self.weights is formatted as a list of layers, with each layer containing lists of random ints.
        :param layerSizes determines number of neurons in each layer
        :param learningRate: for SGD
        '''
        self.numLayers = len(layerSizes)
        self.layerSizes = layerSizes
        self.weights = []
        self.biases = [numpy.random.randn(y, 1).flatten() for y in layerSizes[1:]]
        self.weights = [numpy.random.randn(y, x)
                        for x, y in zip(layerSizes[:-1], layerSizes[1:])]
        self.learningRate = learningRate

    def propogate(self, inp):
        '''
        propogate the network with input list
        '''
        assert len(inp) == self.layerSizes[0], "incorrect input size of " + str(len(inp))
        layers = [inp]
        for bias, weight in zip(self.biases, self.weights):
            layers.append(self.sigmoid(numpy.dot(weight, layers[-1]) + bias))
        return layers
    def train(self, inp, batch_size=100, epochs=10):
        self.gradient_descent(inp, batch_size, epochs)

    def gradient_descent(self, inputs, batch_size, epochs):
        '''
        divide inputs into batches of BATCH_SIZE length and perform GD

        '''
        shuffle(inputs)
        batches = self.partition(inputs, batch_size)
        for batch in batches:
            for f in range(epochs):
                for inp in batch:
                    expected = inp[1]
                    gradients = self.backpropogate(inp[0], expected)
                    biasGradient = gradients[0]
                    self.biases = numpy.subtract(self.biases, numpy.multiply(biasGradient, self.learningRate))
                    weightsGradient = gradients[1]
                    for layer in range(len(self.weights)):
                        for neuron in range(len(self.weights[layer])):
                            self.weights[layer][neuron] = numpy.subtract(self.weights[layer][neuron], numpy.multiply(weightsGradient[layer][neuron], self.learningRate))

    def observed(self, inp):
        output = self.propogate(inp)[2]
        maximum = 0
        for i in range(len(output)):
            if output[i] > output[maximum]:
                maximum = i
        return maximum


    def backpropogate(self, inp, expected):
        '''
        adjust network's weights by SGD
        :param input
        :param expected is the target values
        '''
        biasgradient = [numpy.zeros(b.shape) for b in self.biases]
        weightgradient = [numpy.zeros(w.shape) for w in self.weights]
        layers = [inp]
        act = inp
        zlayers = []
        for bias, weight in zip(self.biases, self.weights):
            aZ = numpy.dot(weight, act)
            aZ += bias
            zlayers.append(aZ)
            act = self.sigmoid(aZ)
            layers.append(act)


        deltacost = self.dcost(layers[-1], expected)
        sigmoidprime = self.dsigmoid(zlayers[-1])
        lastdeltaerror = numpy.multiply(deltacost, sigmoidprime)
        biasgradient[-1] = lastdeltaerror

        weightgradient[-1] = numpy.dot(lastdeltaerror.reshape(len(lastdeltaerror), 1), layers[-2].reshape(1, self.layerSizes[-2]))


        for i in range(2, len(layers) - 1):
            prevLayer = numpy.array(layers[-i-1]).reshape(1, self.layerSizes[-i-1])
            currWeights = self.weights[-i+1]
            lastdeltaerror = numpy.dot(currWeights.transpose(), lastdeltaerror) * self.dsigmoid(zlayers[-i])
            biasgradient[-i] = lastdeltaerror

            weightgradient[-i] = numpy.dot(lastdeltaerror, prevLayer.transpose())
        return [biasgradient, weightgradient]

    def cost(self, observed, expected):
        '''
        The least square cost function for this network's current state.
        '''
        return (observed - expected)
    def dcost(self, observed, expected):
        if (observed.shape == (10,30)):
            return True
        return numpy.subtract(observed, expected)

    def dsigmoid(self, x):
        return self.sigmoid(x) * (1 - self.sigmoid(x))

    def sigmoid(self, x):
        return 1.0 / (1.0 + numpy.exp(-x))

    def partition(self, data, size):
        return [data[i : i + size] for i in range(0, len(data), size)]