U
    ¯Ê¹já	  ã                   @   sú   d Z ddlZddlmZ ddlmZ ejjdedœdd„ƒZ	ejjded	œd
d„ƒZ
G dd„ dejƒZG dd„ dejƒZejjdedœdd„ƒZG dd„ dejƒZejjdedœdd„ƒZG dd„ dejƒZejjd edœdd„ƒZG dd„ dejƒZdS )!aÛ   Activations

A collection of jit-scripted activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

All jit scripted activations are lacking in-place variations on purpose, scripted kernel fusion does not
currently work across in-place op boundaries, thus performance is equal to or less than the non-scripted
versions if they contain in-place ops.

Hacked together by / Copyright 2020 Ross Wightman
é    N)Únn)Ú
functionalF©Úinplacec                 C   s   |   |  ¡ ¡S )z;Swish - Described in: https://arxiv.org/abs/1710.05941
    )ÚmulZsigmoid©Úxr   © r	   úZ/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/layers/activations_jit.pyÚ	swish_jit   s    r   )Ú_inplacec                 C   s   |   t | ¡ ¡ ¡S )ziMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    )r   ÚFZsoftplusÚtanh)r   r   r	   r	   r
   Úmish_jit   s    r   c                       s,   e Zd Zdedœ‡ fdd„Zdd„ Z‡  ZS )ÚSwishJitFr   c                    s   t t| ƒ ¡  d S ©N)Úsuperr   Ú__init__©Úselfr   ©Ú	__class__r	   r
   r   !   s    zSwishJit.__init__c                 C   s   t |ƒS r   )r   ©r   r   r	   r	   r
   Úforward$   s    zSwishJit.forward)F©Ú__name__Ú
__module__Ú__qualname__Úboolr   r   Ú__classcell__r	   r	   r   r
   r       s   r   c                       s,   e Zd Zdedœ‡ fdd„Zdd„ Z‡  ZS )ÚMishJitFr   c                    s   t t| ƒ ¡  d S r   )r   r    r   r   r   r	   r
   r   )   s    zMishJit.__init__c                 C   s   t |ƒS r   )r   r   r	   r	   r
   r   ,   s    zMishJit.forward)Fr   r	   r	   r   r
   r    (   s   r    c                 C   s   | d j ddd� d¡S ©Né   r   é   ©ÚminÚmaxg      @©ÚclampÚdivr   r	   r	   r
   Úhard_sigmoid_jit0   s    r*   c                       s,   e Zd Zdedœ‡ fdd„Zdd„ Z‡  ZS )ÚHardSigmoidJitFr   c                    s   t t| ƒ ¡  d S r   )r   r+   r   r   r   r	   r
   r   7   s    zHardSigmoidJit.__init__c                 C   s   t |ƒS r   )r*   r   r	   r	   r
   r   :   s    zHardSigmoidJit.forward)Fr   r	   r	   r   r
   r+   6   s   r+   c                 C   s   | | d j ddd� d¡ S r!   r'   r   r	   r	   r
   Úhard_swish_jit>   s    r,   c                       s,   e Zd Zdedœ‡ fdd„Zdd„ Z‡  ZS )ÚHardSwishJitFr   c                    s   t t| ƒ ¡  d S r   )r   r-   r   r   r   r	   r
   r   E   s    zHardSwishJit.__init__c                 C   s   t |ƒS r   )r,   r   r	   r	   r
   r   H   s    zHardSwishJit.forward)Fr   r	   r	   r   r
   r-   D   s   r-   c                 C   s   d|  | d j ddd� S )z· Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    g      à?é   r   r$   )r(   r   r	   r	   r
   Úhard_mish_jitL   s    r/   c                       s,   e Zd Zdedœ‡ fdd„Zdd„ Z‡  ZS )ÚHardMishJitFr   c                    s   t t| ƒ ¡  d S r   )r   r0   r   r   r   r	   r
   r   V   s    zHardMishJit.__init__c                 C   s   t |ƒS r   )r/   r   r	   r	   r
   r   Y   s    zHardMishJit.forward)Fr   r	   r	   r   r
   r0   U   s   r0   )F)F)F)F)F)Ú__doc__Ztorchr   Ztorch.nnr   r   ZjitÚscriptr   r   r   ÚModuler   r    r*   r+   r,   r-   r/   r0   r	   r	   r	   r
   Ú<module>   s$   