please add more activation functions

[bionicles]

@jvishnuvardhan @yongtang @seanpmorgan follow-up on the tf issue
System information

• TensorFlow version (you are using): 2b1
• TensorFlow Addons version: pip
• Is it in the tf.contrib (if so, where): idk
• Are you willing to contribute it (yes/no): yes
• Are you willing to maintain it going forward? (yes/no): yes

Describe the feature and the current behavior/state.
activations are high-yield because they dramatically influence performance for little code

Will this change the current api? How?
just adds more activations

Who will benefit with this feature?
people doing hyperparameter search can benefit especially

Any Other info.
here is an updated python file with some activations (converted the if/elif stuff into a lookup table at the bottom)

from tensorflow_addons.activations import sparsemax
import tensorflow as tf

K = tf.keras

B, L = K.backend, K.layers

RRELU_MIN, RRELU_MAX = 0.123, 0.314
HARD_MIN, HARD_MAX = -1., 1.
SOFT_ARGMAX_BETA = 1e10
FN = 'lrelu'


def swish(x):
    """
    Searching for Activation Functions
    https://arxiv.org/abs/1710.05941
    """
    return (B.sigmoid(x) * x)


def soft_argmax(x, beta=SOFT_ARGMAX_BETA):
    """
    https://stackoverflow.com/questions/46926809/getting-around-tf-argmax-which-is-not-differentiable
    https://lucehe.github.io/differentiable-argmax/
    """
    x_range = tf.range(x.shape.as_list()[-1], dtype=x.dtype)
    return tf.math.reduce_sum(
        tf.nn.softmax(x * beta) * x_range, axis=-1)


def gaussian(x):
    return B.exp(-B.pow(x, 2))


def hard_tanh(x, min=HARD_MIN, max=HARD_MAX):
    if x > max:
        return max
    elif x < min:
        return min
    else:
        return x


def hard_lisht(x, min=HARD_MIN, max=HARD_MAX):
    if x < min or x > max:
        return max
    else:
        return tf.math.abs(x)


def lisht(x):
    """
    LiSHT: Non-Parametric Linearly Scaled Hyperbolic Tangent
    https://github.com/swalpa/LiSHT
    """
    return (B.tanh(x) * x)


def rrelu(x, min=RRELU_MIN, max=RRELU_MAX):
    return x if x >= 0 else tf.random.uniform(min, max) * x


def tanh_shrink(x):
    return x - B.tanh(x)


def hard_shrink(x, min=HARD_MIN, max=HARD_MAX):
    if x > max:
        return x
    elif x < min:
        return min
    else:
        return 0


FN_LOOKUP = {
    'soft_argmax': soft_argmax,
    'log_softmax': tf.nn.log_softmax,
    'sparsemax': sparsemax,
    'hard_lisht': hard_lisht,
    'hard_shrink': hard_shrink,
    'tanh_shrink': tanh_shrink,
    'hard_lisht': hard_lisht,
    'hard_tanh': hard_tanh,
    'gaussian': gaussian,
    'swish': swish,
    'lisht': lisht,
    'rrelu': rrelu,
    'lrelu': tf.nn.leaky_relu,
    'crelu': tf.nn.crelu,
    'relu6': tf.nn.relu6,
    'sin': tf.math.sin,
    'cos': tf.math.cos,
}


def clean_activation(activation):
    if callable(activation):
        return activation
    else:
        fn = FN_LOOKUP[activation]
    return fn


def use_fn(fn):
    if not fn:
        fn = FN
    fn = clean_activation(fn)
    return L.Activation(fn)

[seanpmorgan]

Thanks @bionicles! So happy to accept PRs for activations such as swish, lisht, etc. I'm less sold on the value of aliasing tf.math.sin and the other built in ops. Is the rationale just that users may not know they can utilize these ops as activations?

[kyleabeauchamp]

I guess one nice behavior is being able to reference activations as strings rather than functions, which is mostly a convenience but still useful for reducing boilerplate when doing hyperparameter tuning.

[bionicles]

@seanpmorgan @kyleabeauchamp updated the code. yeah, for our architecture search project it's handy to just use strings, but yes, we can directly pass those functions

import tensorflow as tf
K = tf.keras
B, L = K.backend, K.layers

LOWER_ASYMPTOTE = 0
UPPER_ASYMPTOTE_AKA_CARRYING_CAPACITY = 1.
GROWTH_RATE = 1.
LOCATION_OF_MAX_GROWTH = 1.
START_TIME = 0.
COEFFICIENT_OF_EXPONENTIAL_TERM = 1.
IS_RELATED_TO_VALUE_Y_ZERO = 1.
IS_ADDED_TO_EXPONENTIAL_TERM = 1.


def generalized_logistic(
        x,
        a=LOWER_ASYMPTOTE,
        k=UPPER_ASYMPTOTE_AKA_CARRYING_CAPACITY,
        b=GROWTH_RATE,
        q=IS_RELATED_TO_VALUE_Y_ZERO,
        c=IS_ADDED_TO_EXPONENTIAL_TERM,
        m=START_TIME,
        v=LOCATION_OF_MAX_GROWTH,
        ):
    numerator = k - a
    exponential_term = B.exp(-b * (x - m))
    denominator = (c + q * exponential_term ** (1/v))
    return a + numerator / denominator


class Logistic(L.Layer):
    def __init__(self):
        super(Logistic, self).__init__()

    def build(self, input_shape):
        self.lower_asymptote = tf.Variable(
            0., trainable=True)
        self.upper_asymptote_aka_carrying_capacity = tf.Variable(
            1., trainable=True)
        self.growth_rate = tf.Variable(
            1., trainable=True)
        self.is_related_to_value_y_zero = tf.Variable(
            1., trainable=True)
        self.is_added_to_exponential_term = tf.Variable(
            1., trainable=True)
        self.start_time = tf.Variable(
            1., trainable=True)
        self.location_of_max_growth = tf.Variable(
            1., trainable=True)

    def call(self, x):
        return generalized_logistic(
                x,
                a=self.lower_asymptote,
                k=self.upper_asymptote_aka_carrying_capacity,
                b=self.growth_rate,
                q=self.is_related_to_value_y_zero,
                c=self.is_added_to_exponential_term,
                m=self.start_time,
                v=self.location_of_max_growth)

[bionicles]

def mish(x):
    """
    Mish: A Self Regularized Non-Monotonic Neural Activation Function
    https://arxiv.org/abs/1908.08681v1
    """
    return (x * B.tanh(B.softplus(x)))

[fsx950223]

Please assign rrelu to me and it seems swish has been implemented in tensorflow.nn module.@seanpmorgan

[bionicles]

tensorflow/tensorflow#32783

from math import pi
B = tf.keras.backend


SQRT_2_D_PI = B.sqrt(2 / tf.convert_to_tensor(pi))


@tf.function
def gelu(x):
    right = B.tanh(SQRT_2_D_PI * (x + 0.044715 * B.pow(x, 3)))
    return 0.5 * x * (1 + right)

[bionicles]

here are parametric linear, polynomial, and a parametric swish: (tends to blow up and make NaN tho)

import tensorflow as tf

from nature import L1L2

L = tf.keras.layers


class Linear(L.Layer):
    """ y = mx + b
    broadcast scalar weight and bias to all inputs (trainable)
    """

    def __init__(self):
        super().__init__()
        self.m = self.add_weight(
            initializer=tf.keras.initializers.ones(),
            regularizer=L1L2(), trainable=True)
        self.b = self.add_weight(
            initializer="glorot_normal",
            regularizer=L1L2(), trainable=True)

    @tf.function
    def call(self, x):
        return self.m * x + self.b

import tensorflow as tf

from nature import L1L2

init = tf.keras.initializers.TruncatedNormal


class Polynomial(tf.keras.layers.Layer):

    def __init__(self, power=4):
        super().__init__()
        self.powers = []
        for p in list(range(power)):
            coefficient = self.add_weight(
                initializer=init(), trainable=True, regularizer=L1L2())
            super().__setattr__(f"{p}", coefficient)
            self.powers.append((coefficient, p))
        self.built = True

    @tf.function
    def call(self, x):
        y = 0.
        for coefficient, power in self.powers:
            y = y + coefficient * tf.math.pow(x, power)
        return y

import tensorflow as tf

from nature import Polynomial, Logistic, Linear

L = tf.keras.layers


class PSwish(L.Layer):
    def __init__(self, layer_fn=Linear):
        super().__init__()
        self.multiply = L.Multiply()
        self.logistic = Logistic()
        self.linear_or_polynomial = layer_fn()
        self.built = True

    @tf.function
    def call(self, x):
        one = self.linear_or_polynomial(x)
        two = self.logistic(x)
        return self.multiply([one, two])


def PolySwish():
    return PSwish(layer_fn=Polynomial)