mish added

neural_network/activation_functions/mish_activation.py

@@ -0,0 +1,48 @@
+"""
+Implements the Mish activation functions.
+
+The function takes a vector of K real numbers input and then
+applies the Mish function, x*tanh(softplus(x) to each element of the vector.
+
+Script inspired from its corresponding Wikipedia article
+https://en.wikipedia.org/wiki/Rectifier_(neural_networks)
+
+The proposed paper link is provided below.
+https://arxiv.org/abs/1908.08681
+"""
+
+import numpy as np
+
+
+def mish_activation(vector: np.ndarray) -> np.ndarray:
+    """
+        Implements the Mish function
+
+        Parameters:
+            vector: np.array
+
+        Returns:
+            Mish (np.array): The input numpy array after applying tanh.
+
+        mathematically, mish = x * tanh(softplus(x)  where
+        softplus = ln(1+e^(x)) and tanh = (e^x - e^(-x))/(e^x + e^(-x))
+        so, mish can be written as x * (2/(1+e^(-2 * softplus))-1
+
+    Examples:
+        >>> mish_activation(np.array([1,5,6,-0.67]))
+        array([ 0.86509839,  8.99955208, 10.99992663, -1.93211787])
+
+
+        >>> mish_activation(np.array([8,2,-0.98,13]))
+        array([14.9999982 ,  2.94395896, -2.28214659, 25.        ])
+
+
+    """
+    soft_plus = np.log(np.exp(vector) + 1)
+    return vector * (2 / (1 + np.exp(-2 * soft_plus))) - 1
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

neural_network/activation_functions/sigmoid_linear_unit.py

@@ -1,39 +0,0 @@
-"""
-Implements the Sigmoid Linear Unit or SiLU function
-also known as Swiss functions.
-
-The function takes a vector of K real numbers and then
-applies the SiLU function to each element of the vector.
-
-Script inspired from its corresponding Wikipedia article
-https://en.wikipedia.org/wiki/Rectifier_(neural_networks)
-"""
-
-import numpy as np
-
-
-def sigmoid_linear_unit(vector: np.ndarray) -> np.ndarray:
-    """
-         Implements the SiLU activation function.
-         Parameters:
-             vector: the array containing input of SiLU activation
-         return:
-            The input numpy array after applying SiLU.
-
-         Mathematically, f(x) = x *  1/1+e^(-x)
-
-    Examples:
-    >>> sigmoid_linear_unit(vector=np.array([2.3,0.6,-2,-3.8]))
-    array([ 2.09041719,  0.38739378, -0.23840584, -0.08314883])
-
-    >>> sigmoid_linear_unit(vector=np.array([-9.2,-0.3,0.45,-4.56]))
-    array([-0.00092947, -0.12766724,  0.27478766, -0.04721304])
-
-    """
-    return vector / (1 + np.exp(-vector))
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod()