Toy Neural Network

這三段程式碼實現了一個簡單的人工神經網路。它包含以下幾個部分:

1. Matrix.py提供了一個Matrix類別,用於表示和操作矩陣。它實現了常見的矩陣運算,如加法、乘法、轉置等。

2. nn.py定義了一個ActivationFunction類,用於定義激活函數及其導數。它內置了sigmoid和tanh激活函數。還定義了一個NeuralNetwork類,它是一個前饋神經網路,可以進行預測和訓練。

3. toy_neural_network.pyde是主程式,它創建了一個簡單的2-4-1層的神經網路(2個輸入節點、4個隱藏層節點、1個輸出節點)。然後使用XOR數據集訓練該網路5000次迭代。最後,它對訓練後的網路進行預測,並輸出結果。

這段程式碼的作用是實現一個簡單的人工神經網路,能夠學習XOR邏輯門。XOR是一個經典的機器學習問題,常用於測試算法和網路的學習能力。

通過訓練,神經網路應該能夠正確預測XOR的輸出。即對於輸入[0, 0]和[1, 1],輸出為0;對於輸入[0, 1]和[1, 0],輸出為1。

這個程式演示了如何構建一個簡單的前饋神經網路,以及如何使用反向傳播算法訓練該網路。它包含了處理矩陣計算、激活函數、前向傳播和反向傳播等基本元素。雖然非常簡單,但展示了神經網路的基本工作原理。

我主要參考了The Coding Train的這個playlist，再將其改成Processing for Python版本:

toy_neural_network.pyde:

​x
1
from Matrix import *
2
from nn import *
3

4
def setup():
5
    size(400, 400)
6

7
    # test the NeuralNetwork class
8
    nn = NeuralNetwork(2, 4, 1)
9
    inputs = [1, 0]
10
    print("NN prediction before training:")
11
    print(nn.predict(inputs))
12

13
    # train the network with XOR data
14
    for i in range(5000):
15
        nn.train([0, 0], [0])
16
        nn.train([0, 1], [1])
17
        nn.train([1, 0], [1])
18
        nn.train([1, 1], [0])
19
    
20
    
21
    # test the trained network
22
    print("NN prediction after training:")
23
    print(nn.predict([0, 0]))
24
    print(nn.predict([0, 1]))
25
    print(nn.predict([1, 0]))
26
    print(nn.predict([1, 1]))
27

28
    print("all pass")
29

30

31
def draw():
32
    pass

Matrix.py:

xxxxxxxxxx
158
1
import random
2

3
class Matrix():
4
    
5
    def __init__(self, rows, columns):
6
        self.rows = rows
7
        self.columns = columns
8
        self.data = [[0 for x in range(columns)] for y in range(rows)]
9

10
    def _print(self):
11
        for row in self.data:
12
            print(row)
13
        print('\n')
14

15
    def copy(self):
16
        m = Matrix(self.rows, self.columns)
17
        for i in range(self.rows):
18
            for j in range(self.columns):
19
                m.data[i][j] = self.data[i][j]
20
        return m
21
    
22
    @classmethod
23
    def fromArray(cls, array):
24
        m = Matrix(len(array), 1)
25
        for i in range(len(array)):
26
            m.data[i][0] = array[i]
27
        return m
28
    
29
    def toArray(self):
30
        array = []
31
        for i in range(self.rows):
32
            for j in range(self.columns):
33
                array.append(self.data[i][j])
34
        return array
35
    
36
    def add(self, n, b=None):
37
            if b is None:
38
                if isinstance(n, Matrix):
39
                    if self.rows != n.rows or self.columns != n.columns :
40
                        print("Columns and Rows of A must match Columns and Rows of B.")
41
                        return
42
                    result = Matrix(self.rows, self.columns)
43
                    for i in range(self.rows):
44
                        for j in range(self.columns):
45
                            result.data[i][j] = self.data[i][j] + n.data[i][j]
46
                else:
47
                    result = Matrix(self.rows, self.columns)
48
                    for i in range(self.rows):
49
                        for j in range(self.columns):
50
                            result.data[i][j] = self.data[i][j] + n
51
            else:
52
                if isinstance(n, Matrix) and isinstance(b, Matrix):
53
                    if n.rows != b.rows or n.columns != b.columns:
54
                        print("Columns and Rows of A, B and C must match.")
55
                        return
56
                    result = Matrix(n.rows, n.columns)
57
                    for i in range(n.rows):
58
                        for j in range(n.columns):
59
                            result.data[i][j] = n.data[i][j] + b.data[i][j]
60
                elif isinstance(n, Matrix) and not isinstance(b, Matrix):
61
                    result = Matrix(n.rows, n.columns)
62
                    for i in range(n.rows):
63
                        for j in range(n.columns):
64
                            result.data[i][j] = n.data[i][j] + b
65
            return result
66
    
67
    def subtract(self, n, b = None):
68
        if b is None:
69
            if isinstance(n, Matrix):
70
                return self.add(n.multiply(-1))
71
            else:
72
                return self.add(-n)
73
        else:
74
            if isinstance(n, Matrix) and isinstance(b, Matrix):
75
                return n.add(b.multiply(-1))
76
            elif isinstance(n, Matrix) and not isinstance(b, Matrix): 
77
                return n.add(-b)
78
    
79
    
80
    def randomize(self):
81
        for i in range(self.rows):
82
            for j in range(self.columns):
83
                self.data[i][j] = random.uniform(-1, 1)
84

85
    def transpose(self):
86
        result = Matrix(self.columns, self.rows)
87
        for i in range(self.rows):
88
            for j in range(self.columns):
89
                result.data[j][i] = self.data[i][j]
90
        return result
91
    
92
    def multiply(self, n, b = None):
93
        if b is None:
94
            # check if n is a matrix or a scalar
95
            if isinstance(n, Matrix):
96
                if self.columns != n.rows:
97
                    print("Columns of A must match rows of B.")
98
                    return
99
                result = Matrix(self.rows, n.columns)
100
                for i in range(result.rows):
101
                    for j in range(result.columns):
102
                        sum = 0
103
                        for k in range(self.columns):
104
                            sum += self.data[i][k] * n.data[k][j]
105
                        result.data[i][j] = sum
106
                return result
107
            else:
108
                result = Matrix(self.rows, self.columns)
109
                for i in range(self.rows):
110
                    for j in range(self.columns):
111
                        result.data[i][j] = self.data[i][j] * n
112
                return result
113
        else:
114
            if isinstance(n, Matrix) and isinstance(b, Matrix):
115
                if n.columns != b.rows:
116
                    print("Columns of A must match rows of B.")
117
                    return
118
                result = Matrix(n.rows, b.columns)
119
                for i in range(result.rows):
120
                    for j in range(result.columns):
121
                        sum = 0
122
                        for k in range(n.columns):
123
                            sum += n.data[i][k] * b.data[k][j]
124
                        result.data[i][j] = sum
125
                return result
126
            elif isinstance(n, Matrix) and not isinstance(b, Matrix):
127
                result = Matrix(n.rows, n.columns)
128
                for i in range(n.rows):
129
                    for j in range(n.columns):
130
                        result.data[i][j] = n.data[i][j] * b
131
                return result
132

133
    def hadamard_product(self, n, b = None):
134
        if b is None:
135
            if self.rows != n.rows or self.columns != n.columns:
136
                print("Columns and Rows of A must match Columns and Rows of B.")
137
                return
138
            result = Matrix(self.rows, self.columns)
139
            for i in range(result.rows):
140
                for j in range(result.columns):
141
                    result.data[i][j] = self.data[i][j] * n.data[i][j]
142
            return result
143
        else:
144
            if n.rows != b.rows or n.columns != b.columns:
145
                print("Columns and Rows of A, B and C must match.")
146
                return
147
            result = Matrix(n.rows, n.columns)
148
            for i in range(result.rows):
149
                for j in range(result.columns):
150
                    result.data[i][j] = n.data[i][j] * b.data[i][j]
151
            return result
152
        
153
    def map(self, func):
154
        for i in range(self.rows):
155
            for j in range(self.columns):
156
                val = self.data[i][j]
157
                self.data[i][j] = func(val)
158
        return self

nn.py:

xxxxxxxxxx
134
1
from Matrix import *
2
import math
3

4
class ActivationFunction:
5
    def __init__(self, func, dfunc):
6
        self.func = func
7
        self.dfunc = dfunc
8

9
def sigmoid_func(x):
10
    return 1 / (1 + math.exp(-x))
11

12
def sigmoid_dfunc(y):
13
    return y * (1 - y)
14

15
sigmoid = ActivationFunction(sigmoid_func, sigmoid_dfunc)
16

17
def tanh_func(x):
18
    return math.tanh(x)
19

20
def tanh_dfunc(y):
21
    return 1 - (y * y)
22

23
tanh = ActivationFunction(tanh_func, tanh_dfunc)
24

25
class NeuralNetwork:
26
    def __init__(self, a, b=None, c=None):
27
        if isinstance(a, NeuralNetwork):
28
            self.input_nodes = a.input_nodes
29
            self.hidden_nodes = a.hidden_nodes
30
            self.output_nodes = a.output_nodes
31

32
            self.weights_ih = a.weights_ih.copy()
33
            self.weights_ho = a.weights_ho.copy()
34

35
            self.bias_h = a.bias_h.copy()
36
            self.bias_o = a.bias_o.copy()
37
        else:
38
            self.input_nodes = a
39
            self.hidden_nodes = b
40
            self.output_nodes = c
41

42
            self.weights_ih = Matrix(self.hidden_nodes, self.input_nodes)
43
            self.weights_ho = Matrix(self.output_nodes, self.hidden_nodes)
44
            self.weights_ih.randomize()
45
            self.weights_ho.randomize()
46

47
            self.bias_h = Matrix(self.hidden_nodes, 1)
48
            self.bias_o = Matrix(self.output_nodes, 1)
49
            self.bias_h.randomize()
50
            self.bias_o.randomize()
51

52
        self.setLearningRate(0.1)
53
        self.setActivationFunction(sigmoid)
54
    
55
    def predict(self, input_array):
56
        # Generating the Hidden Outputs
57
        inputs = Matrix.fromArray(input_array)
58
        hidden = Matrix.multiply(self.weights_ih, inputs)
59
        hidden = hidden.add(self.bias_h)
60
        # activation function!
61
        hidden.map(self.activation_function.func)
62

63
        # Generating the output's output!
64
        output = Matrix.multiply(self.weights_ho, hidden)
65
        output = output.add(self.bias_o)
66
        output.map(self.activation_function.func)
67

68
        # Sending back to the caller!
69
        return output.toArray()
70

71
    def setLearningRate(self, learning_rate):
72
        self.learning_rate = learning_rate
73

74
    def setActivationFunction(self, func):
75
        self.activation_function = func
76

77
    def train(self, input_array, target_array):
78
        # Generating the Hidden Outputs
79
        inputs = Matrix.fromArray(input_array)
80
        hidden = Matrix.multiply(self.weights_ih, inputs)
81
        hidden = hidden.add(self.bias_h)
82
        # activation function!
83
        hidden.map(self.activation_function.func)
84
        # Generating the output's output!
85
        outputs = Matrix.multiply(self.weights_ho, hidden)
86
        outputs = outputs.add(self.bias_o)
87
        outputs.map(self.activation_function.func)
88

89
        # Convert array to matrix object
90
        targets = Matrix.fromArray(target_array)
91

92
        # Calculate the error
93
        # ERROR = TARGETS - OUTPUTS
94
        output_errors = Matrix.subtract(targets, outputs)
95

96
        # let gradient = outputs * (1 - outputs);
97
        # Calculate gradient
98
        gradients = Matrix.map(outputs, self.activation_function.dfunc)
99
        gradients = gradients.hadamard_product(output_errors)
100
        gradients = gradients.multiply(self.learning_rate)
101

102
        # Calculate deltas
103
        hidden_T = Matrix.transpose(hidden)
104
        weight_ho_deltas = Matrix.multiply(gradients, hidden_T)
105
        # Adjust the weights by deltas
106
        self.weights_ho = self.weights_ho.add(weight_ho_deltas)
107
        # Adjust the bias by its deltas (which is just the gradients)
108
        self.bias_o = self.bias_o.add(gradients)
109

110
        # Calculate the hidden layer errors
111
        who_t = Matrix.transpose(self.weights_ho)
112
        hidden_errors = Matrix.multiply(who_t, output_errors)
113
      
114
        # Calculate hidden gradient
115
        hidden_gradient = Matrix.map(hidden, self.activation_function.dfunc)
116
        hidden_gradient = hidden_gradient.hadamard_product(hidden_errors)
117
        hidden_gradient = hidden_gradient.multiply(self.learning_rate)
118

119
        # Calcuate input->hidden deltas
120
        inputs_T = Matrix.transpose(inputs)
121
        weight_ih_deltas = Matrix.multiply(hidden_gradient, inputs_T)
122

123
        self.weights_ih = self.weights_ih.add(weight_ih_deltas)
124
        # Adjust the bias by its deltas (which is just the gradients)
125
        self.bias_h = self.bias_h.add(hidden_gradient)
126

127
    def copy(self):
128
        return NeuralNetwork(self)
129
    
130
    def mutate(self, func):
131
        self.weights_ih.map(func)
132
        self.weights_ho.map(func)
133
        self.bias_o.map(func)
134
        self.bias_h.map(func)