FFN Activation Functions › Implementation: Configurable Activation Module

class Activation:
    """
    Configurable activation function for FFN layers.

    Supports ReLU, GELU (exact and approximate), and SiLU.
    """

    ACTIVATIONS = {
        "relu": lambda x: np.maximum(0, x),
        "gelu": lambda x: x * 0.5 * (1 + erf(x / np.sqrt(2))),
        "gelu_approximate": lambda x: (
            0.5 * x * (1 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * x**3)))
        ),
        "silu": lambda x: x * (1 / (1 + np.exp(-np.clip(x, -500, 500)))),
    }

    def __init__(self, activation_type="gelu"):
        """
        Initialize the activation function.

        Args:
            activation_type: One of "relu", "gelu", "gelu_approximate", "silu"
        """
        if activation_type not in self.ACTIVATIONS:
            raise ValueError(
                f"Unknown activation: {activation_type}. "
                f"Choose from {list(self.ACTIVATIONS.keys())}"
            )

        self.activation_type = activation_type
        self._func = self.ACTIVATIONS[activation_type]

    def __call__(self, x):
        """Apply the activation function."""
        return self._func(x)

    def __repr__(self):
        return f"Activation({self.activation_type})"


# Test the module
test_input = np.array([-2, -1, 0, 1, 2])