add a framework of bp neural network and delete the old one

Neural_Network/bpnn.py

@@ -0,0 +1,190 @@
+'''
+
+A Framework of Back Propagation Neural Network（BP） model
+
+Easy to use:
+    * add many layers as you want ！！！
+    * clearly see how the loss decreasing
+Easy to expand:
+    * more activation functions
+    * more loss functions
+    * more optimization method
+
+Author: Stephen Lee
+Github : https://github.com/RiptideBo
+Date: 2017.11.23
+
+'''
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+def sigmoid(x):
+    return 1 / (1 + np.exp(-1 * x))
+
+class DenseLayer():
+    '''
+    Layers of BP neural network
+    '''
+    def __init__(self,units,activation=None,learning_rate=None,is_input_layer=False):
+        '''
+        common connected layer of bp network
+        :param units: numbers of neural units
+        :param activation: activation function
+        :param learning_rate: learning rate for paras
+        :param is_input_layer: whether it is input layer or not
+        '''
+        self.units = units
+        self.weight = None
+        self.bias = None
+        self.activation = activation
+        if learning_rate is None:
+            learning_rate = 0.3
+        self.learn_rate = learning_rate
+        self.is_input_layer = is_input_layer
+
+    def initializer(self,back_units):
+        self.weight = np.asmatrix(np.random.normal(0,0.5,(self.units,back_units)))
+        self.bias = np.asmatrix(np.random.normal(0,0.5,self.units)).T
+        if self.activation is None:
+            self.activation = sigmoid
+
+    def cal_gradient(self):
+        if self.activation == sigmoid:
+            gradient_mat = np.dot(self.output ,(1- self.output).T)
+            gradient_activation = np.diag(np.diag(gradient_mat))
+        else:
+            gradient_activation = 1
+        return gradient_activation
+
+    def forward_propagation(self,xdata):
+        self.xdata = xdata
+        if self.is_input_layer:
+            # input layer
+            self.wx_plus_b = xdata
+            self.output = xdata
+            return xdata
+        else:
+            self.wx_plus_b = np.dot(self.weight,self.xdata) - self.bias
+            self.output = self.activation(self.wx_plus_b)
+            return self.output
+
+    def back_propagation(self,gradient):
+
+        gradient_activation = self.cal_gradient() # i * i 维
+        gradient = np.asmatrix(np.dot(gradient.T,gradient_activation))
+
+        self._gradient_weight = np.asmatrix(self.xdata)
+        self._gradient_bias = -1
+        self._gradient_x = self.weight
+
+        self.gradient_weight = np.dot(gradient.T,self._gradient_weight.T)
+        self.gradient_bias = gradient * self._gradient_bias
+        self.gradient = np.dot(gradient,self._gradient_x).T
+        # ----------------------upgrade
+        # -----------the Negative gradient direction --------
+        self.weight = self.weight - self.learn_rate * self.gradient_weight
+        self.bias = self.bias - self.learn_rate * self.gradient_bias.T
+
+        return self.gradient
+
+
+class BPNN():
+    '''
+    Back Propagation Neural Network model
+    '''
+    def __init__(self):
+        self.layers = []
+        self.train_mse = []
+        self.fig_loss = plt.figure()
+        self.ax_loss = self.fig_loss.add_subplot(1,1,1)
+
+    def add_layer(self,layer):
+        self.layers.append(layer)
+
+    def build(self):
+        for i,layer in enumerate(self.layers[:]):
+            if i < 1:
+                layer.is_input_layer = True
+            else:
+                layer.initializer(self.layers[i-1].units)
+
+    def summary(self):
+        for i,layer in enumerate(self.layers[:]):
+            print('------- layer %d -------'%i)
+            print('weight.shape ',np.shape(layer.weight))
+            print('bias.shape ',np.shape(layer.bias))
+
+    def train(self,xdata,ydata,train_round,accuracy):
+        self.train_round = train_round
+        self.accuracy = accuracy
+
+        self.ax_loss.hlines(self.accuracy, 0, self.train_round * 1.1)
+
+        x_shape = np.shape(xdata)
+        for round_i in range(train_round):
+            all_loss = 0
+            for row in range(x_shape[0]):
+                _xdata = np.asmatrix(xdata[row,:]).T
+                _ydata = np.asmatrix(ydata[row,:]).T
+
+                # forward propagation
+                for layer in self.layers:
+                    _xdata = layer.forward_propagation(_xdata)
+
+                loss, gradient = self.cal_loss(_ydata, _xdata)
+                all_loss = all_loss + loss
+
+                # back propagation
+                # the input_layer does not upgrade
+                for layer in self.layers[:0:-1]:
+                    gradient = layer.back_propagation(gradient)
+
+            mse = all_loss/x_shape[0]
+            self.train_mse.append(mse)
+
+            self.plot_loss()
+
+            if mse < self.accuracy:
+                print('----达到精度----')
+                return mse
+
+    def cal_loss(self,ydata,ydata_):
+        self.loss = np.sum(np.power((ydata - ydata_),2))
+        self.loss_gradient = 2 * (ydata_ - ydata)
+        # vector (shape is the same as _ydata.shape)
+        return self.loss,self.loss_gradient
+
+    def plot_loss(self):
+        if self.ax_loss.lines:
+            self.ax_loss.lines.remove(self.ax_loss.lines[0])
+        self.ax_loss.plot(self.train_mse, 'r-')
+        plt.ion()
+        plt.show()
+        plt.pause(0.1)
+
+
+
+
+def example():
+
+    x = np.random.randn(10,10)
+    y = np.asarray([[0.8,0.4],[0.4,0.3],[0.34,0.45],[0.67,0.32],
+                    [0.88,0.67],[0.78,0.77],[0.55,0.66],[0.55,0.43],[0.54,0.1],
+                    [0.1,0.5]])
+
+    model = BPNN()
+    model.add_layer(DenseLayer(10))
+    model.add_layer(DenseLayer(20))
+    model.add_layer(DenseLayer(30))
+    model.add_layer(DenseLayer(2))
+
+    model.build()
+
+    model.summary()
+
+    model.train(xdata=x,ydata=y,train_round=100,accuracy=0.01)
+
+if __name__ == '__main__':
+    example()

Neural_Network/neuralnetwork_bp3.py

@@ -1,152 +0,0 @@
-#-*- coding:utf-8 -*-
-'''
-Author: Stephen Lee
-Date: 2017.9.21
-
-BP neural network with three layers
-'''
-
-import numpy as np
-import matplotlib.pyplot as plt
-
-class Bpnn():
-
-    def __init__(self,n_layer1,n_layer2,n_layer3,rate_w=0.3,rate_t=0.3):
-        '''
-        :param n_layer1: number of input layer
-        :param n_layer2: number of hiden layer
-        :param n_layer3: number of output layer
-        :param rate_w: rate of weight learning
-        :param rate_t: rate of threshold learning
-        '''
-        self.num1 = n_layer1
-        self.num2 = n_layer2
-        self.num3 = n_layer3
-        self.rate_weight = rate_w
-        self.rate_thre = rate_t
-        self.thre2 = -2*np.random.rand(self.num2)+1
-        self.thre3 = -2*np.random.rand(self.num3)+1
-        self.vji = np.mat(-2*np.random.rand(self.num2, self.num1)+1)
-        self.wkj = np.mat(-2*np.random.rand(self.num3, self.num2)+1)
-
-    def sig(self,x):
-        return 1 / (1 + np.exp(-1*x))
-
-    def sig_plain(self,x):
-        return 1 / (1 + np.exp(-1*x))
-
-    def do_round(self,x):
-        return round(x, 3)
-
-    def trian(self,patterns,data_train, data_teach, n_repeat, error_accuracy, draw_e=False):
-        '''
-        :param patterns: the number of patterns
-        :param data_train: training data x; numpy.ndarray
-        :param data_teach: training data y; numpy.ndarray
-        :param n_repeat: echoes
-        :param error_accuracy: error accuracy
-        :return: None
-        '''
-        data_train = np.asarray(data_train)
-        data_teach = np.asarray(data_teach)
-        # print('-------------------Start Training-------------------------')
-        # print(' - - Shape: Train_Data  ',np.shape(data_train))
-        # print(' - - Shape: Teach_Data  ',np.shape(data_teach))
-        rp = 0
-        all_mse = []
-        mse  = 10000
-        while rp < n_repeat and mse >= error_accuracy:
-            alle = 0
-            final_out = []
-            for g in range(np.shape(data_train)[0]):
-                net_i = data_train[g]
-                out1 = net_i
-
-                net_j = out1 * self.vji.T  - self.thre2
-                out2=self.sig(net_j)
-
-                net_k = out2 * self.wkj.T  - self.thre3
-                out3 = self.sig(net_k)
-
-                # learning process
-                pd_k_all = np.multiply(np.multiply(out3,(1 - out3)),(data_teach[g]-out3))
-                pd_j_all = np.multiply(pd_k_all * self.wkj,np.multiply(out2,1-out2))
-                #upgrade weight
-                self.wkj = self.wkj + pd_k_all.T * out2 *self.rate_weight
-                self.vji = self.vji + pd_j_all.T * out1 * self.rate_weight
-                #upgrade threshold
-                self.thre3 = self.thre3 - pd_k_all * self.rate_thre
-                self.thre2 = self.thre2 - pd_j_all * self.rate_thre
-                #calculate sum of error
-                errors = np.sum(abs((data_teach[g] - out3)))
-
-                alle = alle + errors
-                final_out.extend(out3.getA().tolist())
-            final_out3 = [list(map(self.do_round,each)) for each in final_out]
-
-            rp = rp + 1
-            mse = alle/patterns
-            all_mse.append(mse)
-        def draw_error():
-            yplot = [error_accuracy for i in range(int(n_repeat * 1.2))]
-            plt.plot(all_mse, '+-')
-            plt.plot(yplot, 'r--')
-            plt.xlabel('Learning Times')
-            plt.ylabel('All_mse')
-            plt.grid(True,alpha = 0.7)
-            plt.show()
-        # print('------------------Training Complished---------------------')
-        # print(' - - Training epoch: ', rp, '     - - Mse: %.6f'%mse)
-        # print(' - - Last Output: ', final_out3)
-        if draw_e:
-            draw_error()
-
-    def predict(self,data_test):
-        '''
-        :param data_test: data test, numpy.ndarray
-        :return: predict output data
-        '''
-        data_test = np.asarray(data_test)
-        produce_out = []
-        # print('-------------------Start Testing-------------------------')
-        # print(' - - Shape: Test_Data  ',np.shape(data_test))
-        # print(np.shape(data_test))
-        for g in range(np.shape(data_test)[0]):
-
-            net_i = data_test[g]
-            out1 = net_i
-
-            net_j = out1 * self.vji.T - self.thre2
-            out2 = self.sig(net_j)
-
-            net_k = out2 * self.wkj.T - self.thre3
-            out3 = self.sig(net_k)
-            produce_out.extend(out3.getA().tolist())
-        res = [list(map(self.do_round,each)) for each in produce_out]
-        return np.asarray(res)
-
-
-def main():
-    #example data
-    data_x = [[1,2,3,4],
-              [5,6,7,8],
-              [2,2,3,4],
-              [7,7,8,8]]
-    data_y = [[1,0,0,0],
-              [0,1,0,0],
-              [0,0,1,0],
-              [0,0,0,1]]
-
-    test_x = [[1,2,3,4],
-              [3,2,3,4]]
-
-    #building network model
-    model = Bpnn(4,10,4)
-    #training the model
-    model.trian(patterns=4,data_train=data_x,data_teach=data_y,
-                n_repeat=100,error_accuracy=0.1,draw_e=True)
-    #predicting data
-    model.predict(test_x)
-
-if __name__ == '__main__':
-    main()