mirror of
https://github.com/TheAlgorithms/Python.git
synced 2024-11-27 23:11:09 +00:00
add a framework of bp neural network and delete the old one
This commit is contained in:
RiptideBo
parent
a03b2eafc0
commit
d7a94a1135
190
Neural_Network/bpnn.py
Normal file
190
Neural_Network/bpnn.py
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
'''
|
||||
|
||||
A Framework of Back Propagation Neural Network（BP） model
|
||||
|
||||
Easy to use:
|
||||
* add many layers as you want ！！！
|
||||
* clearly see how the loss decreasing
|
||||
Easy to expand:
|
||||
* more activation functions
|
||||
* more loss functions
|
||||
* more optimization method
|
||||
|
||||
Author: Stephen Lee
|
||||
Github : https://github.com/RiptideBo
|
||||
Date: 2017.11.23
|
||||
|
||||
'''
|
||||
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def sigmoid(x):
|
||||
return 1 / (1 + np.exp(-1 * x))
|
||||
|
||||
class DenseLayer():
|
||||
'''
|
||||
Layers of BP neural network
|
||||
'''
|
||||
def __init__(self,units,activation=None,learning_rate=None,is_input_layer=False):
|
||||
'''
|
||||
common connected layer of bp network
|
||||
:param units: numbers of neural units
|
||||
:param activation: activation function
|
||||
:param learning_rate: learning rate for paras
|
||||
:param is_input_layer: whether it is input layer or not
|
||||
'''
|
||||
self.units = units
|
||||
self.weight = None
|
||||
self.bias = None
|
||||
self.activation = activation
|
||||
if learning_rate is None:
|
||||
learning_rate = 0.3
|
||||
self.learn_rate = learning_rate
|
||||
self.is_input_layer = is_input_layer
|
||||
|
||||
def initializer(self,back_units):
|
||||
self.weight = np.asmatrix(np.random.normal(0,0.5,(self.units,back_units)))
|
||||
self.bias = np.asmatrix(np.random.normal(0,0.5,self.units)).T
|
||||
if self.activation is None:
|
||||
self.activation = sigmoid
|
||||
|
||||
def cal_gradient(self):
|
||||
if self.activation == sigmoid:
|
||||
gradient_mat = np.dot(self.output ,(1- self.output).T)
|
||||
gradient_activation = np.diag(np.diag(gradient_mat))
|
||||
else:
|
||||
gradient_activation = 1
|
||||
return gradient_activation
|
||||
|
||||
def forward_propagation(self,xdata):
|
||||
self.xdata = xdata
|
||||
if self.is_input_layer:
|
||||
# input layer
|
||||
self.wx_plus_b = xdata
|
||||
self.output = xdata
|
||||
return xdata
|
||||
else:
|
||||
self.wx_plus_b = np.dot(self.weight,self.xdata) - self.bias
|
||||
self.output = self.activation(self.wx_plus_b)
|
||||
return self.output
|
||||
|
||||
def back_propagation(self,gradient):
|
||||
|
||||
gradient_activation = self.cal_gradient() # i * i 维
|
||||
gradient = np.asmatrix(np.dot(gradient.T,gradient_activation))
|
||||
|
||||
self._gradient_weight = np.asmatrix(self.xdata)
|
||||
self._gradient_bias = -1
|
||||
self._gradient_x = self.weight
|
||||
|
||||
self.gradient_weight = np.dot(gradient.T,self._gradient_weight.T)
|
||||
self.gradient_bias = gradient * self._gradient_bias
|
||||
self.gradient = np.dot(gradient,self._gradient_x).T
|
||||
# ----------------------upgrade
|
||||
# -----------the Negative gradient direction --------
|
||||
self.weight = self.weight - self.learn_rate * self.gradient_weight
|
||||
self.bias = self.bias - self.learn_rate * self.gradient_bias.T
|
||||
|
||||
return self.gradient
|
||||
|
||||
|
||||
class BPNN():
|
||||
'''
|
||||
Back Propagation Neural Network model
|
||||
'''
|
||||
def __init__(self):
|
||||
self.layers = []
|
||||
self.train_mse = []
|
||||
self.fig_loss = plt.figure()
|
||||
self.ax_loss = self.fig_loss.add_subplot(1,1,1)
|
||||
|
||||
def add_layer(self,layer):
|
||||
self.layers.append(layer)
|
||||
|
||||
def build(self):
|
||||
for i,layer in enumerate(self.layers[:]):
|
||||
if i < 1:
|
||||
layer.is_input_layer = True
|
||||
else:
|
||||
layer.initializer(self.layers[i-1].units)
|
||||
|
||||
def summary(self):
|
||||
for i,layer in enumerate(self.layers[:]):
|
||||
print('------- layer %d -------'%i)
|
||||
print('weight.shape ',np.shape(layer.weight))
|
||||
print('bias.shape ',np.shape(layer.bias))
|
||||
|
||||
def train(self,xdata,ydata,train_round,accuracy):
|
||||
self.train_round = train_round
|
||||
self.accuracy = accuracy
|
||||
|
||||
self.ax_loss.hlines(self.accuracy, 0, self.train_round * 1.1)
|
||||
|
||||
x_shape = np.shape(xdata)
|
||||
for round_i in range(train_round):
|
||||
all_loss = 0
|
||||
for row in range(x_shape[0]):
|
||||
_xdata = np.asmatrix(xdata[row,:]).T
|
||||
_ydata = np.asmatrix(ydata[row,:]).T
|
||||
|
||||
# forward propagation
|
||||
for layer in self.layers:
|
||||
_xdata = layer.forward_propagation(_xdata)
|
||||
|
||||
loss, gradient = self.cal_loss(_ydata, _xdata)
|
||||
all_loss = all_loss + loss
|
||||
|
||||
# back propagation
|
||||
# the input_layer does not upgrade
|
||||
for layer in self.layers[:0:-1]:
|
||||
gradient = layer.back_propagation(gradient)
|
||||
|
||||
mse = all_loss/x_shape[0]
|
||||
self.train_mse.append(mse)
|
||||
|
||||
self.plot_loss()
|
||||
|
||||
if mse < self.accuracy:
|
||||
print('----达到精度----')
|
||||
return mse
|
||||
|
||||
def cal_loss(self,ydata,ydata_):
|
||||
self.loss = np.sum(np.power((ydata - ydata_),2))
|
||||
self.loss_gradient = 2 * (ydata_ - ydata)
|
||||
# vector (shape is the same as _ydata.shape)
|
||||
return self.loss,self.loss_gradient
|
||||
|
||||
def plot_loss(self):
|
||||
if self.ax_loss.lines:
|
||||
self.ax_loss.lines.remove(self.ax_loss.lines[0])
|
||||
self.ax_loss.plot(self.train_mse, 'r-')
|
||||
plt.ion()
|
||||
plt.show()
|
||||
plt.pause(0.1)
|
||||
|
||||
|
||||
|
||||
|
||||
def example():
|
||||
|
||||
x = np.random.randn(10,10)
|
||||
y = np.asarray([[0.8,0.4],[0.4,0.3],[0.34,0.45],[0.67,0.32],
|
||||
[0.88,0.67],[0.78,0.77],[0.55,0.66],[0.55,0.43],[0.54,0.1],
|
||||
[0.1,0.5]])
|
||||
|
||||
model = BPNN()
|
||||
model.add_layer(DenseLayer(10))
|
||||
model.add_layer(DenseLayer(20))
|
||||
model.add_layer(DenseLayer(30))
|
||||
model.add_layer(DenseLayer(2))
|
||||
|
||||
model.build()
|
||||
|
||||
model.summary()
|
||||
|
||||
model.train(xdata=x,ydata=y,train_round=100,accuracy=0.01)
|
||||
|
||||
if __name__ == '__main__':
|
||||
example()
|
||||
|
|
@ -1,152 +0,0 @@
|
|||
#-*- coding:utf-8 -*-
|
||||
'''
|
||||
Author: Stephen Lee
|
||||
Date: 2017.9.21
|
||||
|
||||
BP neural network with three layers
|
||||
'''
|
||||
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
class Bpnn():
|
||||
|
||||
def __init__(self,n_layer1,n_layer2,n_layer3,rate_w=0.3,rate_t=0.3):
|
||||
'''
|
||||
:param n_layer1: number of input layer
|
||||
:param n_layer2: number of hiden layer
|
||||
:param n_layer3: number of output layer
|
||||
:param rate_w: rate of weight learning
|
||||
:param rate_t: rate of threshold learning
|
||||
'''
|
||||
self.num1 = n_layer1
|
||||
self.num2 = n_layer2
|
||||
self.num3 = n_layer3
|
||||
self.rate_weight = rate_w
|
||||
self.rate_thre = rate_t
|
||||
self.thre2 = -2*np.random.rand(self.num2)+1
|
||||
self.thre3 = -2*np.random.rand(self.num3)+1
|
||||
self.vji = np.mat(-2*np.random.rand(self.num2, self.num1)+1)
|
||||
self.wkj = np.mat(-2*np.random.rand(self.num3, self.num2)+1)
|
||||
|
||||
def sig(self,x):
|
||||
return 1 / (1 + np.exp(-1*x))
|
||||
|
||||
def sig_plain(self,x):
|
||||
return 1 / (1 + np.exp(-1*x))
|
||||
|
||||
def do_round(self,x):
|
||||
return round(x, 3)
|
||||
|
||||
def trian(self,patterns,data_train, data_teach, n_repeat, error_accuracy, draw_e=False):
|
||||
'''
|
||||
:param patterns: the number of patterns
|
||||
:param data_train: training data x; numpy.ndarray
|
||||
:param data_teach: training data y; numpy.ndarray
|
||||
:param n_repeat: echoes
|
||||
:param error_accuracy: error accuracy
|
||||
:return: None
|
||||
'''
|
||||
data_train = np.asarray(data_train)
|
||||
data_teach = np.asarray(data_teach)
|
||||
# print('-------------------Start Training-------------------------')
|
||||
# print(' - - Shape: Train_Data ',np.shape(data_train))
|
||||
# print(' - - Shape: Teach_Data ',np.shape(data_teach))
|
||||
rp = 0
|
||||
all_mse = []
|
||||
mse = 10000
|
||||
while rp < n_repeat and mse >= error_accuracy:
|
||||
alle = 0
|
||||
final_out = []
|
||||
for g in range(np.shape(data_train)[0]):
|
||||
net_i = data_train[g]
|
||||
out1 = net_i
|
||||
|
||||
net_j = out1 * self.vji.T - self.thre2
|
||||
out2=self.sig(net_j)
|
||||
|
||||
net_k = out2 * self.wkj.T - self.thre3
|
||||
out3 = self.sig(net_k)
|
||||
|
||||
# learning process
|
||||
pd_k_all = np.multiply(np.multiply(out3,(1 - out3)),(data_teach[g]-out3))
|
||||
pd_j_all = np.multiply(pd_k_all * self.wkj,np.multiply(out2,1-out2))
|
||||
#upgrade weight
|
||||
self.wkj = self.wkj + pd_k_all.T * out2 *self.rate_weight
|
||||
self.vji = self.vji + pd_j_all.T * out1 * self.rate_weight
|
||||
#upgrade threshold
|
||||
self.thre3 = self.thre3 - pd_k_all * self.rate_thre
|
||||
self.thre2 = self.thre2 - pd_j_all * self.rate_thre
|
||||
#calculate sum of error
|
||||
errors = np.sum(abs((data_teach[g] - out3)))
|
||||
|
||||
alle = alle + errors
|
||||
final_out.extend(out3.getA().tolist())
|
||||
final_out3 = [list(map(self.do_round,each)) for each in final_out]
|
||||
|
||||
rp = rp + 1
|
||||
mse = alle/patterns
|
||||
all_mse.append(mse)
|
||||
def draw_error():
|
||||
yplot = [error_accuracy for i in range(int(n_repeat * 1.2))]
|
||||
plt.plot(all_mse, '+-')
|
||||
plt.plot(yplot, 'r--')
|
||||
plt.xlabel('Learning Times')
|
||||
plt.ylabel('All_mse')
|
||||
plt.grid(True,alpha = 0.7)
|
||||
plt.show()
|
||||
# print('------------------Training Complished---------------------')
|
||||
# print(' - - Training epoch: ', rp, ' - - Mse: %.6f'%mse)
|
||||
# print(' - - Last Output: ', final_out3)
|
||||
if draw_e:
|
||||
draw_error()
|
||||
|
||||
def predict(self,data_test):
|
||||
'''
|
||||
:param data_test: data test, numpy.ndarray
|
||||
:return: predict output data
|
||||
'''
|
||||
data_test = np.asarray(data_test)
|
||||
produce_out = []
|
||||
# print('-------------------Start Testing-------------------------')
|
||||
# print(' - - Shape: Test_Data ',np.shape(data_test))
|
||||
# print(np.shape(data_test))
|
||||
for g in range(np.shape(data_test)[0]):
|
||||
|
||||
net_i = data_test[g]
|
||||
out1 = net_i
|
||||
|
||||
net_j = out1 * self.vji.T - self.thre2
|
||||
out2 = self.sig(net_j)
|
||||
|
||||
net_k = out2 * self.wkj.T - self.thre3
|
||||
out3 = self.sig(net_k)
|
||||
produce_out.extend(out3.getA().tolist())
|
||||
res = [list(map(self.do_round,each)) for each in produce_out]
|
||||
return np.asarray(res)
|
||||
|
||||
|
||||
def main():
|
||||
#example data
|
||||
data_x = [[1,2,3,4],
|
||||
[5,6,7,8],
|
||||
[2,2,3,4],
|
||||
[7,7,8,8]]
|
||||
data_y = [[1,0,0,0],
|
||||
[0,1,0,0],
|
||||
[0,0,1,0],
|
||||
[0,0,0,1]]
|
||||
|
||||
test_x = [[1,2,3,4],
|
||||
[3,2,3,4]]
|
||||
|
||||
#building network model
|
||||
model = Bpnn(4,10,4)
|
||||
#training the model
|
||||
model.trian(patterns=4,data_train=data_x,data_teach=data_y,
|
||||
n_repeat=100,error_accuracy=0.1,draw_e=True)
|
||||
#predicting data
|
||||
model.predict(test_x)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user