Python/neural_network/back_propagation_neural_network.py

200 lines
5.9 KiB
Python
Raw Normal View History

2018-10-19 12:48:28 +00:00
#!/usr/bin/python
2019-10-05 05:14:13 +00:00
"""
2018-10-19 12:48:28 +00:00
A Framework of Back Propagation Neural NetworkBP model
Easy to use:
* add many layers as you want
* clearly see how the loss decreasing
Easy to expand:
* more activation functions
* more loss functions
* more optimization method
Author: Stephen Lee
Github : https://github.com/RiptideBo
Date: 2017.11.23
2019-10-05 05:14:13 +00:00
"""
2018-10-19 12:48:28 +00:00
import numpy as np
from matplotlib import pyplot as plt
2018-10-19 12:48:28 +00:00
def sigmoid(x):
return 1 / (1 + np.exp(-1 * x))
2019-10-05 05:14:13 +00:00
class DenseLayer:
"""
2018-10-19 12:48:28 +00:00
Layers of BP neural network
2019-10-05 05:14:13 +00:00
"""
2019-10-05 05:14:13 +00:00
def __init__(
self, units, activation=None, learning_rate=None, is_input_layer=False
):
"""
2018-10-19 12:48:28 +00:00
common connected layer of bp network
:param units: numbers of neural units
:param activation: activation function
:param learning_rate: learning rate for paras
:param is_input_layer: whether it is input layer or not
2019-10-05 05:14:13 +00:00
"""
2018-10-19 12:48:28 +00:00
self.units = units
self.weight = None
self.bias = None
self.activation = activation
if learning_rate is None:
learning_rate = 0.3
self.learn_rate = learning_rate
self.is_input_layer = is_input_layer
2019-10-05 05:14:13 +00:00
def initializer(self, back_units):
self.weight = np.asmatrix(np.random.normal(0, 0.5, (self.units, back_units)))
self.bias = np.asmatrix(np.random.normal(0, 0.5, self.units)).T
2018-10-19 12:48:28 +00:00
if self.activation is None:
self.activation = sigmoid
def cal_gradient(self):
# activation function may be sigmoid or linear
2018-10-19 12:48:28 +00:00
if self.activation == sigmoid:
2019-10-05 05:14:13 +00:00
gradient_mat = np.dot(self.output, (1 - self.output).T)
2018-10-19 12:48:28 +00:00
gradient_activation = np.diag(np.diag(gradient_mat))
else:
gradient_activation = 1
return gradient_activation
2019-10-05 05:14:13 +00:00
def forward_propagation(self, xdata):
2018-10-19 12:48:28 +00:00
self.xdata = xdata
if self.is_input_layer:
# input layer
self.wx_plus_b = xdata
self.output = xdata
return xdata
else:
2019-10-05 05:14:13 +00:00
self.wx_plus_b = np.dot(self.weight, self.xdata) - self.bias
2018-10-19 12:48:28 +00:00
self.output = self.activation(self.wx_plus_b)
return self.output
2019-10-05 05:14:13 +00:00
def back_propagation(self, gradient):
gradient_activation = self.cal_gradient() # i * i 维
gradient = np.asmatrix(np.dot(gradient.T, gradient_activation))
2018-10-19 12:48:28 +00:00
self._gradient_weight = np.asmatrix(self.xdata)
self._gradient_bias = -1
self._gradient_x = self.weight
2019-10-05 05:14:13 +00:00
self.gradient_weight = np.dot(gradient.T, self._gradient_weight.T)
2018-10-19 12:48:28 +00:00
self.gradient_bias = gradient * self._gradient_bias
2019-10-05 05:14:13 +00:00
self.gradient = np.dot(gradient, self._gradient_x).T
# upgrade: the Negative gradient direction
2018-10-19 12:48:28 +00:00
self.weight = self.weight - self.learn_rate * self.gradient_weight
self.bias = self.bias - self.learn_rate * self.gradient_bias.T
# updates the weights and bias according to learning rate (0.3 if undefined)
2018-10-19 12:48:28 +00:00
return self.gradient
2019-10-05 05:14:13 +00:00
class BPNN:
"""
2018-10-19 12:48:28 +00:00
Back Propagation Neural Network model
2019-10-05 05:14:13 +00:00
"""
2018-10-19 12:48:28 +00:00
def __init__(self):
self.layers = []
self.train_mse = []
self.fig_loss = plt.figure()
2019-10-05 05:14:13 +00:00
self.ax_loss = self.fig_loss.add_subplot(1, 1, 1)
2018-10-19 12:48:28 +00:00
2019-10-05 05:14:13 +00:00
def add_layer(self, layer):
2018-10-19 12:48:28 +00:00
self.layers.append(layer)
def build(self):
2019-10-05 05:14:13 +00:00
for i, layer in enumerate(self.layers[:]):
2018-10-19 12:48:28 +00:00
if i < 1:
layer.is_input_layer = True
else:
2019-10-05 05:14:13 +00:00
layer.initializer(self.layers[i - 1].units)
2018-10-19 12:48:28 +00:00
def summary(self):
2019-10-05 05:14:13 +00:00
for i, layer in enumerate(self.layers[:]):
print(f"------- layer {i} -------")
2019-10-05 05:14:13 +00:00
print("weight.shape ", np.shape(layer.weight))
print("bias.shape ", np.shape(layer.bias))
2018-10-19 12:48:28 +00:00
2019-10-05 05:14:13 +00:00
def train(self, xdata, ydata, train_round, accuracy):
2018-10-19 12:48:28 +00:00
self.train_round = train_round
self.accuracy = accuracy
self.ax_loss.hlines(self.accuracy, 0, self.train_round * 1.1)
x_shape = np.shape(xdata)
Add flake8 pluin flake8 bugbear to pre-commit (#7132) * ci(pre-commit): Add ``flake8-builtins`` additional dependency to ``pre-commit`` (#7104) * refactor: Fix ``flake8-builtins`` (#7104) * fix(lru_cache): Fix naming conventions in docstrings (#7104) * ci(pre-commit): Order additional dependencies alphabetically (#7104) * fix(lfu_cache): Correct function name in docstring (#7104) * Update strings/snake_case_to_camel_pascal_case.py Co-authored-by: Christian Clauss <cclauss@me.com> * Update data_structures/stacks/next_greater_element.py Co-authored-by: Christian Clauss <cclauss@me.com> * Update digital_image_processing/index_calculation.py Co-authored-by: Christian Clauss <cclauss@me.com> * Update graphs/prim.py Co-authored-by: Christian Clauss <cclauss@me.com> * Update hashes/djb2.py Co-authored-by: Christian Clauss <cclauss@me.com> * refactor: Rename `_builtin` to `builtin_` ( #7104) * fix: Rename all instances (#7104) * refactor: Update variable names (#7104) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * ci: Create ``tox.ini`` and ignore ``A003`` (#7123) * revert: Remove function name changes (#7104) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Rename tox.ini to .flake8 * Update data_structures/heap/heap.py Co-authored-by: Dhruv Manilawala <dhruvmanila@gmail.com> * refactor: Rename `next_` to `next_item` (#7104) * ci(pre-commit): Add `flake8` plugin `flake8-bugbear` (#7127) * refactor: Follow `flake8-bugbear` plugin (#7127) * fix: Correct `knapsack` code (#7127) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: Christian Clauss <cclauss@me.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Dhruv Manilawala <dhruvmanila@gmail.com>
2022-10-13 16:03:06 +00:00
for _ in range(train_round):
2018-10-19 12:48:28 +00:00
all_loss = 0
for row in range(x_shape[0]):
2019-10-05 05:14:13 +00:00
_xdata = np.asmatrix(xdata[row, :]).T
_ydata = np.asmatrix(ydata[row, :]).T
2018-10-19 12:48:28 +00:00
# forward propagation
for layer in self.layers:
_xdata = layer.forward_propagation(_xdata)
loss, gradient = self.cal_loss(_ydata, _xdata)
all_loss = all_loss + loss
# back propagation: the input_layer does not upgrade
2018-10-19 12:48:28 +00:00
for layer in self.layers[:0:-1]:
gradient = layer.back_propagation(gradient)
2019-10-05 05:14:13 +00:00
mse = all_loss / x_shape[0]
2018-10-19 12:48:28 +00:00
self.train_mse.append(mse)
self.plot_loss()
if mse < self.accuracy:
2019-10-05 05:14:13 +00:00
print("----达到精度----")
2018-10-19 12:48:28 +00:00
return mse
2019-10-05 05:14:13 +00:00
def cal_loss(self, ydata, ydata_):
self.loss = np.sum(np.power((ydata - ydata_), 2))
2018-10-19 12:48:28 +00:00
self.loss_gradient = 2 * (ydata_ - ydata)
# vector (shape is the same as _ydata.shape)
2019-10-05 05:14:13 +00:00
return self.loss, self.loss_gradient
2018-10-19 12:48:28 +00:00
def plot_loss(self):
if self.ax_loss.lines:
self.ax_loss.lines.remove(self.ax_loss.lines[0])
2019-10-05 05:14:13 +00:00
self.ax_loss.plot(self.train_mse, "r-")
2018-10-19 12:48:28 +00:00
plt.ion()
2019-10-05 05:14:13 +00:00
plt.xlabel("step")
plt.ylabel("loss")
2018-10-19 12:48:28 +00:00
plt.show()
plt.pause(0.1)
def example():
2019-10-05 05:14:13 +00:00
x = np.random.randn(10, 10)
y = np.asarray(
[
[0.8, 0.4],
[0.4, 0.3],
[0.34, 0.45],
[0.67, 0.32],
[0.88, 0.67],
[0.78, 0.77],
[0.55, 0.66],
[0.55, 0.43],
[0.54, 0.1],
[0.1, 0.5],
]
)
2018-10-19 12:48:28 +00:00
model = BPNN()
for i in (10, 20, 30, 2):
model.add_layer(DenseLayer(i))
2018-10-19 12:48:28 +00:00
model.build()
model.summary()
2019-10-05 05:14:13 +00:00
model.train(xdata=x, ydata=y, train_round=100, accuracy=0.01)
2018-10-19 12:48:28 +00:00
2019-10-05 05:14:13 +00:00
if __name__ == "__main__":
2018-10-19 12:48:28 +00:00
example()