From 3a671b57a29e3c2b4e973b01bc5bbe1554aa5da2 Mon Sep 17 00:00:00 2001
From: Kuldeep Borkar <74557588+KuldeepBorkar@users.noreply.github.com>
Date: Fri, 28 Oct 2022 19:57:16 +0530
Subject: [PATCH] Implemented Swish Function (#7357)

* Implemented Swish Function

* Added more description and return hint in def

* Changed the name and added more descrition including test for sigmoid function

* Added * in front of links
---
 maths/sigmoid_linear_unit.py | 57 ++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 maths/sigmoid_linear_unit.py

diff --git a/maths/sigmoid_linear_unit.py b/maths/sigmoid_linear_unit.py
new file mode 100644
index 000000000..a8ada10dd
--- /dev/null
+++ b/maths/sigmoid_linear_unit.py
@@ -0,0 +1,57 @@
+"""
+This script demonstrates the implementation of the Sigmoid Linear Unit (SiLU)
+or swish function.
+* https://en.wikipedia.org/wiki/Rectifier_(neural_networks)
+* https://en.wikipedia.org/wiki/Swish_function
+
+The function takes a vector x of K real numbers as input and returns x * sigmoid(x).
+Swish is a smooth, non-monotonic function defined as f(x) = x * sigmoid(x).
+Extensive experiments shows that Swish consistently matches or outperforms ReLU
+on deep networks applied to a variety of challenging domains such as
+image classification and machine translation.
+
+This script is inspired by a corresponding research paper.
+* https://arxiv.org/abs/1710.05941
+"""
+
+import numpy as np
+
+
+def sigmoid(vector: np.array) -> np.array:
+    """
+    Mathematical function sigmoid takes a vector x of K real numbers as input and
+    returns 1/ (1 + e^-x).
+    https://en.wikipedia.org/wiki/Sigmoid_function
+
+    >>> sigmoid(np.array([-1.0, 1.0, 2.0]))
+    array([0.26894142, 0.73105858, 0.88079708])
+    """
+    return 1 / (1 + np.exp(-vector))
+
+
+def sigmoid_linear_unit(vector: np.array) -> np.array:
+    """
+    Implements the Sigmoid Linear Unit (SiLU) or swish function
+
+    Parameters:
+        vector (np.array): A  numpy array consisting of real
+        values.
+
+    Returns:
+        swish_vec (np.array): The input numpy array, after applying
+        swish.
+
+    Examples:
+    >>> sigmoid_linear_unit(np.array([-1.0, 1.0, 2.0]))
+    array([-0.26894142,  0.73105858,  1.76159416])
+
+    >>> sigmoid_linear_unit(np.array([-2]))
+    array([-0.23840584])
+    """
+    return vector * sigmoid(vector)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()