Add 2-hidden layer neural network with back propagation using sigmoid activation function (#4037)

* added neural network with 2 hidden layers

* Revert "added neural network with 2 hidden layers"

This reverts commit fa4e2ac86e.

* added neural network with 2 hidden layers

* passing pre-commit requirements

* doctest completed

* added return hints

* added example

* example added

* completed doctest's

* changes made as per the review

* changes made

* changes after review

* changes

* spacing

* changed return type

* changed dtype

neural_network/2_hidden_layers_neural_network.py

@@ -0,0 +1,295 @@
+"""
+References:
+    - http://neuralnetworksanddeeplearning.com/chap2.html (Backpropagation)
+    - https://en.wikipedia.org/wiki/Sigmoid_function (Sigmoid activation function)
+    - https://en.wikipedia.org/wiki/Feedforward_neural_network (Feedforward)
+"""
+
+import numpy
+
+
+class TwoHiddenLayerNeuralNetwork:
+    def __init__(self, input_array: numpy.ndarray, output_array: numpy.ndarray) -> None:
+        """
+        This function initializes the TwoHiddenLayerNeuralNetwork class with random
+        weights for every layer and initializes predicted output with zeroes.
+
+        input_array : input values for training the neural network (i.e training data) .
+        output_array : expected output values of the given inputs.
+        """
+
+        # Input values provided for training the model.
+        self.input_array = input_array
+
+        # Random initial weights are assigned where first argument is the
+        # number of nodes in previous layer and second argument is the
+        # number of nodes in the next layer.
+
+        # Random initial weights are assigned.
+        # self.input_array.shape[1] is used to represent number of nodes in input layer.
+        # First hidden layer consists of 4 nodes.
+        self.input_layer_and_first_hidden_layer_weights = numpy.random.rand(
+            self.input_array.shape[1], 4
+        )
+
+        # Random initial values for the first hidden layer.
+        # First hidden layer has 4 nodes.
+        # Second hidden layer has 3 nodes.
+        self.first_hidden_layer_and_second_hidden_layer_weights = numpy.random.rand(
+            4, 3
+        )
+
+        # Random initial values for the second hidden layer.
+        # Second hidden layer has 3 nodes.
+        # Output layer has 1 node.
+        self.second_hidden_layer_and_output_layer_weights = numpy.random.rand(3, 1)
+
+        # Real output values provided.
+        self.output_array = output_array
+
+        # Predicted output values by the neural network.
+        # Predicted_output array initially consists of zeroes.
+        self.predicted_output = numpy.zeros(output_array.shape)
+
+    def feedforward(self) -> numpy.ndarray:
+        """
+        The information moves in only one direction i.e. forward from the input nodes,
+        through the two hidden nodes and to the output nodes.
+        There are no cycles or loops in the network.
+
+        Return layer_between_second_hidden_layer_and_output
+            (i.e the last layer of the neural network).
+
+        >>> input_val = numpy.array(([0, 0, 0], [0, 0, 0], [0, 0, 0]), dtype=float)
+        >>> output_val = numpy.array(([0], [0], [0]), dtype=float)
+        >>> nn = TwoHiddenLayerNeuralNetwork(input_val, output_val)
+        >>> res = nn.feedforward()
+        >>> array_sum = numpy.sum(res)
+        >>> numpy.isnan(array_sum)
+        False
+        """
+        # Layer_between_input_and_first_hidden_layer is the layer connecting the
+        # input nodes with the first hidden layer nodes.
+        self.layer_between_input_and_first_hidden_layer = sigmoid(
+            numpy.dot(self.input_array, self.input_layer_and_first_hidden_layer_weights)
+        )
+
+        # layer_between_first_hidden_layer_and_second_hidden_layer is the layer
+        # connecting the first hidden set of nodes with the second hidden set of nodes.
+        self.layer_between_first_hidden_layer_and_second_hidden_layer = sigmoid(
+            numpy.dot(
+                self.layer_between_input_and_first_hidden_layer,
+                self.first_hidden_layer_and_second_hidden_layer_weights,
+            )
+        )
+
+        # layer_between_second_hidden_layer_and_output is the layer connecting
+        # second hidden layer with the output node.
+        self.layer_between_second_hidden_layer_and_output = sigmoid(
+            numpy.dot(
+                self.layer_between_first_hidden_layer_and_second_hidden_layer,
+                self.second_hidden_layer_and_output_layer_weights,
+            )
+        )
+
+        return self.layer_between_second_hidden_layer_and_output
+
+    def back_propagation(self) -> None:
+        """
+        Function for fine-tuning the weights of the neural net based on the
+        error rate obtained in the previous epoch (i.e., iteration).
+        Updation is done using derivative of sogmoid activation function.
+
+        >>> input_val = numpy.array(([0, 0, 0], [0, 0, 0], [0, 0, 0]), dtype=float)
+        >>> output_val = numpy.array(([0], [0], [0]), dtype=float)
+        >>> nn = TwoHiddenLayerNeuralNetwork(input_val, output_val)
+        >>> res = nn.feedforward()
+        >>> nn.back_propagation()
+        >>> updated_weights = nn.second_hidden_layer_and_output_layer_weights
+        >>> (res == updated_weights).all()
+        False
+        """
+
+        updated_second_hidden_layer_and_output_layer_weights = numpy.dot(
+            self.layer_between_first_hidden_layer_and_second_hidden_layer.T,
+            2
+            * (self.output_array - self.predicted_output)
+            * sigmoid_derivative(self.predicted_output),
+        )
+        updated_first_hidden_layer_and_second_hidden_layer_weights = numpy.dot(
+            self.layer_between_input_and_first_hidden_layer.T,
+            numpy.dot(
+                2
+                * (self.output_array - self.predicted_output)
+                * sigmoid_derivative(self.predicted_output),
+                self.second_hidden_layer_and_output_layer_weights.T,
+            )
+            * sigmoid_derivative(
+                self.layer_between_first_hidden_layer_and_second_hidden_layer
+            ),
+        )
+        updated_input_layer_and_first_hidden_layer_weights = numpy.dot(
+            self.input_array.T,
+            numpy.dot(
+                numpy.dot(
+                    2
+                    * (self.output_array - self.predicted_output)
+                    * sigmoid_derivative(self.predicted_output),
+                    self.second_hidden_layer_and_output_layer_weights.T,
+                )
+                * sigmoid_derivative(
+                    self.layer_between_first_hidden_layer_and_second_hidden_layer
+                ),
+                self.first_hidden_layer_and_second_hidden_layer_weights.T,
+            )
+            * sigmoid_derivative(self.layer_between_input_and_first_hidden_layer),
+        )
+
+        self.input_layer_and_first_hidden_layer_weights += (
+            updated_input_layer_and_first_hidden_layer_weights
+        )
+        self.first_hidden_layer_and_second_hidden_layer_weights += (
+            updated_first_hidden_layer_and_second_hidden_layer_weights
+        )
+        self.second_hidden_layer_and_output_layer_weights += (
+            updated_second_hidden_layer_and_output_layer_weights
+        )
+
+    def train(self, output: numpy.ndarray, iterations: int, give_loss: bool) -> None:
+        """
+        Performs the feedforwarding and back propagation process for the
+        given number of iterations.
+        Every iteration will update the weights of neural network.
+
+        output : real output values,required for calculating loss.
+        iterations : number of times the weights are to be updated.
+        give_loss : boolean value, If True then prints loss for each iteration,
+                    If False then nothing is printed
+
+        >>> input_val = numpy.array(([0, 0, 0], [0, 1, 0], [0, 0, 1]), dtype=float)
+        >>> output_val = numpy.array(([0], [1], [1]), dtype=float)
+        >>> nn = TwoHiddenLayerNeuralNetwork(input_val, output_val)
+        >>> first_iteration_weights = nn.feedforward()
+        >>> nn.back_propagation()
+        >>> updated_weights = nn.second_hidden_layer_and_output_layer_weights
+        >>> (first_iteration_weights == updated_weights).all()
+        False
+        """
+        for iteration in range(1, iterations + 1):
+            self.output = self.feedforward()
+            self.back_propagation()
+            if give_loss:
+                loss = numpy.mean(numpy.square(output - self.feedforward()))
+                print(f"Iteration {iteration} Loss: {loss}")
+
+    def predict(self, input: numpy.ndarray) -> int:
+        """
+        Predict's the output for the given input values using
+        the trained neural network.
+
+        The output value given by the model ranges in-between 0 and 1.
+        The predict function returns 1 if the model value is greater
+        than the threshold value else returns 0,
+        as the real output values are in binary.
+
+        >>> input_val = numpy.array(([0, 0, 0], [0, 1, 0], [0, 0, 1]), dtype=float)
+        >>> output_val = numpy.array(([0], [1], [1]), dtype=float)
+        >>> nn = TwoHiddenLayerNeuralNetwork(input_val, output_val)
+        >>> nn.train(output_val, 1000, False)
+        >>> nn.predict([0,1,0])
+        1
+        """
+
+        # Input values for which the predictions are to be made.
+        self.array = input
+
+        self.layer_between_input_and_first_hidden_layer = sigmoid(
+            numpy.dot(self.array, self.input_layer_and_first_hidden_layer_weights)
+        )
+
+        self.layer_between_first_hidden_layer_and_second_hidden_layer = sigmoid(
+            numpy.dot(
+                self.layer_between_input_and_first_hidden_layer,
+                self.first_hidden_layer_and_second_hidden_layer_weights,
+            )
+        )
+
+        self.layer_between_second_hidden_layer_and_output = sigmoid(
+            numpy.dot(
+                self.layer_between_first_hidden_layer_and_second_hidden_layer,
+                self.second_hidden_layer_and_output_layer_weights,
+            )
+        )
+
+        return int(self.layer_between_second_hidden_layer_and_output > 0.6)
+
+
+def sigmoid(value: numpy.ndarray) -> numpy.ndarray:
+    """
+    Applies sigmoid activation function.
+
+    return normalized values
+
+    >>> sigmoid(numpy.array(([1, 0, 2], [1, 0, 0]), dtype=numpy.float64))
+    array([[0.73105858, 0.5       , 0.88079708],
+           [0.73105858, 0.5       , 0.5       ]])
+    """
+    return 1 / (1 + numpy.exp(-value))
+
+
+def sigmoid_derivative(value: numpy.ndarray) -> numpy.ndarray:
+    """
+    Provides the derivative value of the sigmoid function.
+
+    returns derivative of the sigmoid value
+
+    >>> sigmoid_derivative(numpy.array(([1, 0, 2], [1, 0, 0]), dtype=numpy.float64))
+    array([[ 0.,  0., -2.],
+           [ 0.,  0.,  0.]])
+    """
+    return (value) * (1 - (value))
+
+
+def example() -> int:
+    """
+    Example for "how to use the neural network class and use the
+    respected methods for the desired output".
+    Calls the TwoHiddenLayerNeuralNetwork class and
+    provides the fixed input output values to the model.
+    Model is trained for a fixed amount of iterations then the predict method is called.
+    In this example the output is divided into 2 classes i.e. binary classification,
+    the two classes are represented by '0' and '1'.
+
+    >>> example()
+    1
+    """
+    # Input values.
+    input = numpy.array(
+        (
+            [0, 0, 0],
+            [0, 0, 1],
+            [0, 1, 0],
+            [0, 1, 1],
+            [1, 0, 0],
+            [1, 0, 1],
+            [1, 1, 0],
+            [1, 1, 1],
+        ),
+        dtype=numpy.float64,
+    )
+
+    # True output values for the given input values.
+    output = numpy.array(([0], [1], [1], [0], [1], [0], [0], [1]), dtype=numpy.float64)
+
+    # Calling neural network class.
+    neural_network = TwoHiddenLayerNeuralNetwork(input_array=input, output_array=output)
+
+    # Calling training function.
+    # Set give_loss to True if you want to see loss in every iteration.
+    neural_network.train(output=output, iterations=10, give_loss=False)
+
+    return neural_network.predict(numpy.array(([1, 1, 1]), dtype=numpy.float64))
+
+
+if __name__ == "__main__":
+    example()