From 572de4f15e266057e806a751006156a212a3812e Mon Sep 17 00:00:00 2001 From: Shivansh Bhatnagar Date: Wed, 18 Oct 2023 20:20:18 +0530 Subject: [PATCH] Added A General Swish Activation Function inNeural Networks (#10415) * Added A General Swish Activation Function inNeural Networks * Added the general swish function in the SiLU function and renamed it as swish.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Shivansh Bhatnagar Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../{sigmoid_linear_unit.py => swish.py} | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) rename neural_network/activation_functions/{sigmoid_linear_unit.py => swish.py} (72%) diff --git a/neural_network/activation_functions/sigmoid_linear_unit.py b/neural_network/activation_functions/swish.py similarity index 72% rename from neural_network/activation_functions/sigmoid_linear_unit.py rename to neural_network/activation_functions/swish.py index 0ee09bf82..ab3d8fa12 100644 --- a/neural_network/activation_functions/sigmoid_linear_unit.py +++ b/neural_network/activation_functions/swish.py @@ -12,6 +12,7 @@ image classification and machine translation. This script is inspired by a corresponding research paper. * https://arxiv.org/abs/1710.05941 +* https://blog.paperspace.com/swish-activation-function/ """ import numpy as np @@ -49,6 +50,25 @@ def sigmoid_linear_unit(vector: np.ndarray) -> np.ndarray: return vector * sigmoid(vector) +def swish(vector: np.ndarray, trainable_parameter: int) -> np.ndarray: + """ + Parameters: + vector (np.ndarray): A numpy array consisting of real values + trainable_parameter: Use to implement various Swish Activation Functions + + Returns: + swish_vec (np.ndarray): The input numpy array, after applying swish + + Examples: + >>> swish(np.array([-1.0, 1.0, 2.0]), 2) + array([-0.11920292, 0.88079708, 1.96402758]) + + >>> swish(np.array([-2]), 1) + array([-0.23840584]) + """ + return vector * sigmoid(trainable_parameter * vector) + + if __name__ == "__main__": import doctest