math_utils.py

import numpy as np

"""
Initialization strategies
"""
def uniform_initialization(input_size, output_size, type):
    """
    Initializes weights and biases by sampling from a uniform distribution in the range [-1, 1] and [0, 1] respectively.
    
    """
    if type == "weights":
        #  Create array of size (input_size, output_size) with elements taken from a uniform distribution with range [-1, 1]
        return np.random.uniform(-1, 1, (input_size, output_size))
    elif type == "biases":
        #  Create array of size (input_size, 1) with elements taken from a uniform distribution with range [0, 1]
        return np.random.uniform(0, 1, (input_size, output_size))
    

def normal_initialization(input_size, output_size, type):
    """
    Initializes weights and biases by sampling from a normal distribution with mean 0 and standard deviation 1.
    """
    return np.random.randn(input_size, output_size)


"""
Activation functions
"""
def sigmoidal_logistic(x, alpha=1):
    return 1/(1+np.exp(-alpha*x))


def relu(array):
    return np.maximum(0, array)


def tanh(array):
    return (np.exp(array) - np.exp(-array))/(np.exp(array) + np.exp(-array))


def mish_function(x):
    #  Mish function from Misra 2019 (see report for details)
    return x * tanh(np.log(1+np.exp(x)))

def softplus(x, beta=1):
    #  softplus activation function (see https://pytorch.org/docs/stable/generated/torch.nn.Softplus.html)
    return np.log(1 + np.exp(beta * x)) / beta