diff --git a/Neural_Network/bpnn.py b/Neural_Network/bpnn.py new file mode 100644 index 000000000..ed5d4c8cb --- /dev/null +++ b/Neural_Network/bpnn.py @@ -0,0 +1,190 @@ +''' + +A Framework of Back Propagation Neural Network（BP） model + +Easy to use: + * add many layers as you want ！！！ + * clearly see how the loss decreasing +Easy to expand: + * more activation functions + * more loss functions + * more optimization method + +Author: Stephen Lee +Github : https://github.com/RiptideBo +Date: 2017.11.23 + +''' + +import numpy as np +import matplotlib.pyplot as plt + + +def sigmoid(x): + return 1 / (1 + np.exp(-1 * x)) + +class DenseLayer(): + ''' + Layers of BP neural network + ''' + def __init__(self,units,activation=None,learning_rate=None,is_input_layer=False): + ''' + common connected layer of bp network + :param units: numbers of neural units + :param activation: activation function + :param learning_rate: learning rate for paras + :param is_input_layer: whether it is input layer or not + ''' + self.units = units + self.weight = None + self.bias = None + self.activation = activation + if learning_rate is None: + learning_rate = 0.3 + self.learn_rate = learning_rate + self.is_input_layer = is_input_layer + + def initializer(self,back_units): + self.weight = np.asmatrix(np.random.normal(0,0.5,(self.units,back_units))) + self.bias = np.asmatrix(np.random.normal(0,0.5,self.units)).T + if self.activation is None: + self.activation = sigmoid + + def cal_gradient(self): + if self.activation == sigmoid: + gradient_mat = np.dot(self.output ,(1- self.output).T) + gradient_activation = np.diag(np.diag(gradient_mat)) + else: + gradient_activation = 1 + return gradient_activation + + def forward_propagation(self,xdata): + self.xdata = xdata + if self.is_input_layer: + # input layer + self.wx_plus_b = xdata + self.output = xdata + return xdata + else: + self.wx_plus_b = np.dot(self.weight,self.xdata) - self.bias + self.output = self.activation(self.wx_plus_b) + return self.output + + def back_propagation(self,gradient): + + gradient_activation = self.cal_gradient() # i * i 维 + gradient = np.asmatrix(np.dot(gradient.T,gradient_activation)) + + self._gradient_weight = np.asmatrix(self.xdata) + self._gradient_bias = -1 + self._gradient_x = self.weight + + self.gradient_weight = np.dot(gradient.T,self._gradient_weight.T) + self.gradient_bias = gradient * self._gradient_bias + self.gradient = np.dot(gradient,self._gradient_x).T + # ----------------------upgrade + # -----------the Negative gradient direction -------- + self.weight = self.weight - self.learn_rate * self.gradient_weight + self.bias = self.bias - self.learn_rate * self.gradient_bias.T + + return self.gradient + + +class BPNN(): + ''' + Back Propagation Neural Network model + ''' + def __init__(self): + self.layers = [] + self.train_mse = [] + self.fig_loss = plt.figure() + self.ax_loss = self.fig_loss.add_subplot(1,1,1) + + def add_layer(self,layer): + self.layers.append(layer) + + def build(self): + for i,layer in enumerate(self.layers[:]): + if i < 1: + layer.is_input_layer = True + else: + layer.initializer(self.layers[i-1].units) + + def summary(self): + for i,layer in enumerate(self.layers[:]): + print('------- layer %d -------'%i) + print('weight.shape ',np.shape(layer.weight)) + print('bias.shape ',np.shape(layer.bias)) + + def train(self,xdata,ydata,train_round,accuracy): + self.train_round = train_round + self.accuracy = accuracy + + self.ax_loss.hlines(self.accuracy, 0, self.train_round * 1.1) + + x_shape = np.shape(xdata) + for round_i in range(train_round): + all_loss = 0 + for row in range(x_shape[0]): + _xdata = np.asmatrix(xdata[row,:]).T + _ydata = np.asmatrix(ydata[row,:]).T + + # forward propagation + for layer in self.layers: + _xdata = layer.forward_propagation(_xdata) + + loss, gradient = self.cal_loss(_ydata, _xdata) + all_loss = all_loss + loss + + # back propagation + # the input_layer does not upgrade + for layer in self.layers[:0:-1]: + gradient = layer.back_propagation(gradient) + + mse = all_loss/x_shape[0] + self.train_mse.append(mse) + + self.plot_loss() + + if mse < self.accuracy: + print('----达到精度----') + return mse + + def cal_loss(self,ydata,ydata_): + self.loss = np.sum(np.power((ydata - ydata_),2)) + self.loss_gradient = 2 * (ydata_ - ydata) + # vector (shape is the same as _ydata.shape) + return self.loss,self.loss_gradient + + def plot_loss(self): + if self.ax_loss.lines: + self.ax_loss.lines.remove(self.ax_loss.lines[0]) + self.ax_loss.plot(self.train_mse, 'r-') + plt.ion() + plt.show() + plt.pause(0.1) + + + + +def example(): + + x = np.random.randn(10,10) + y = np.asarray([[0.8,0.4],[0.4,0.3],[0.34,0.45],[0.67,0.32], + [0.88,0.67],[0.78,0.77],[0.55,0.66],[0.55,0.43],[0.54,0.1], + [0.1,0.5]]) + + model = BPNN() + model.add_layer(DenseLayer(10)) + model.add_layer(DenseLayer(20)) + model.add_layer(DenseLayer(30)) + model.add_layer(DenseLayer(2)) + + model.build() + + model.summary() + + model.train(xdata=x,ydata=y,train_round=100,accuracy=0.01) + +if __name__ == '__main__': + example() diff --git a/Neural_Network/neuralnetwork_bp3.py b/Neural_Network/neuralnetwork_bp3.py deleted file mode 100644 index 896411a5d..000000000 --- a/Neural_Network/neuralnetwork_bp3.py +++ /dev/null @@ -1,152 +0,0 @@ -#-*- coding:utf-8 -*- -''' -Author: Stephen Lee -Date: 2017.9.21 - -BP neural network with three layers -''' - -import numpy as np -import matplotlib.pyplot as plt - -class Bpnn(): - - def __init__(self,n_layer1,n_layer2,n_layer3,rate_w=0.3,rate_t=0.3): - ''' - :param n_layer1: number of input layer - :param n_layer2: number of hiden layer - :param n_layer3: number of output layer - :param rate_w: rate of weight learning - :param rate_t: rate of threshold learning - ''' - self.num1 = n_layer1 - self.num2 = n_layer2 - self.num3 = n_layer3 - self.rate_weight = rate_w - self.rate_thre = rate_t - self.thre2 = -2*np.random.rand(self.num2)+1 - self.thre3 = -2*np.random.rand(self.num3)+1 - self.vji = np.mat(-2*np.random.rand(self.num2, self.num1)+1) - self.wkj = np.mat(-2*np.random.rand(self.num3, self.num2)+1) - - def sig(self,x): - return 1 / (1 + np.exp(-1*x)) - - def sig_plain(self,x): - return 1 / (1 + np.exp(-1*x)) - - def do_round(self,x): - return round(x, 3) - - def trian(self,patterns,data_train, data_teach, n_repeat, error_accuracy, draw_e=False): - ''' - :param patterns: the number of patterns - :param data_train: training data x; numpy.ndarray - :param data_teach: training data y; numpy.ndarray - :param n_repeat: echoes - :param error_accuracy: error accuracy - :return: None - ''' - data_train = np.asarray(data_train) - data_teach = np.asarray(data_teach) - # print('-------------------Start Training-------------------------') - # print(' - - Shape: Train_Data ',np.shape(data_train)) - # print(' - - Shape: Teach_Data ',np.shape(data_teach)) - rp = 0 - all_mse = [] - mse = 10000 - while rp < n_repeat and mse >= error_accuracy: - alle = 0 - final_out = [] - for g in range(np.shape(data_train)[0]): - net_i = data_train[g] - out1 = net_i - - net_j = out1 * self.vji.T - self.thre2 - out2=self.sig(net_j) - - net_k = out2 * self.wkj.T - self.thre3 - out3 = self.sig(net_k) - - # learning process - pd_k_all = np.multiply(np.multiply(out3,(1 - out3)),(data_teach[g]-out3)) - pd_j_all = np.multiply(pd_k_all * self.wkj,np.multiply(out2,1-out2)) - #upgrade weight - self.wkj = self.wkj + pd_k_all.T * out2 *self.rate_weight - self.vji = self.vji + pd_j_all.T * out1 * self.rate_weight - #upgrade threshold - self.thre3 = self.thre3 - pd_k_all * self.rate_thre - self.thre2 = self.thre2 - pd_j_all * self.rate_thre - #calculate sum of error - errors = np.sum(abs((data_teach[g] - out3))) - - alle = alle + errors - final_out.extend(out3.getA().tolist()) - final_out3 = [list(map(self.do_round,each)) for each in final_out] - - rp = rp + 1 - mse = alle/patterns - all_mse.append(mse) - def draw_error(): - yplot = [error_accuracy for i in range(int(n_repeat * 1.2))] - plt.plot(all_mse, '+-') - plt.plot(yplot, 'r--') - plt.xlabel('Learning Times') - plt.ylabel('All_mse') - plt.grid(True,alpha = 0.7) - plt.show() - # print('------------------Training Complished---------------------') - # print(' - - Training epoch: ', rp, ' - - Mse: %.6f'%mse) - # print(' - - Last Output: ', final_out3) - if draw_e: - draw_error() - - def predict(self,data_test): - ''' - :param data_test: data test, numpy.ndarray - :return: predict output data - ''' - data_test = np.asarray(data_test) - produce_out = [] - # print('-------------------Start Testing-------------------------') - # print(' - - Shape: Test_Data ',np.shape(data_test)) - # print(np.shape(data_test)) - for g in range(np.shape(data_test)[0]): - - net_i = data_test[g] - out1 = net_i - - net_j = out1 * self.vji.T - self.thre2 - out2 = self.sig(net_j) - - net_k = out2 * self.wkj.T - self.thre3 - out3 = self.sig(net_k) - produce_out.extend(out3.getA().tolist()) - res = [list(map(self.do_round,each)) for each in produce_out] - return np.asarray(res) - - -def main(): - #example data - data_x = [[1,2,3,4], - [5,6,7,8], - [2,2,3,4], - [7,7,8,8]] - data_y = [[1,0,0,0], - [0,1,0,0], - [0,0,1,0], - [0,0,0,1]] - - test_x = [[1,2,3,4], - [3,2,3,4]] - - #building network model - model = Bpnn(4,10,4) - #training the model - model.trian(patterns=4,data_train=data_x,data_teach=data_y, - n_repeat=100,error_accuracy=0.1,draw_e=True) - #predicting data - model.predict(test_x) - -if __name__ == '__main__': - main()