U
    ¯Ê¹jþ  ã                   @   sÌ  d Z ddlZddlmZ ddlmZ ejjdd„ ƒZejjdd„ ƒZ	G d	d
„ d
ej
jƒZd9dd„ZG dd„ dejƒZejjdd„ ƒZejjdd„ ƒZG dd„ dej
jƒZd:dd„ZG dd„ dejƒZejjd;edœdd„ƒZejjdd„ ƒZG dd „ d ej
jƒZd<edœd!d"„ZG d#d$„ d$ejƒZejjd%d&„ ƒZejjd'd(„ ƒZG d)d*„ d*ej
jƒZd=d+d,„ZG d-d.„ d.ejƒZejjd/d0„ ƒZ ejjd1d2„ ƒZ!G d3d4„ d4ej
jƒZ"d>edœd5d6„Z#G d7d8„ d8ejƒZ$dS )?a�   Activations (memory-efficient w/ custom autograd)

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

These activations are not compatible with jit scripting or ONNX export of the model, please use either
the JIT or basic versions of the activations.

Hacked together by / Copyright 2020 Ross Wightman
é    N)Únn)Ú
functionalc                 C   s   |   t | ¡¡S ©N)ÚmulÚtorchÚsigmoid©Úx© r
   úY/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/layers/activations_me.pyÚswish_jit_fwd   s    r   c                 C   s"   t  | ¡}||d| d|     S ©Né   )r   r   )r	   Úgrad_outputÚ	x_sigmoidr
   r
   r   Úswish_jit_bwd   s    
r   c                   @   s4   e Zd ZdZedd„ ƒZedd„ ƒZedd„ ƒZdS )	ÚSwishJitAutoFnzÃ torch.jit.script optimised Swish w/ memory-efficient checkpoint
    Inspired by conversation btw Jeremy Howard & Adam Pazske
    https://twitter.com/jeremyphoward/status/1188251041835315200
    c                 C   s   |   d||   d|¡¡S )NÚMulZSigmoid)Úop)Úgr	   r
   r
   r   Úsymbolic!   s    zSwishJitAutoFn.symbolicc                 C   s   |   |¡ t|ƒS r   )Úsave_for_backwardr   ©Úctxr	   r
   r
   r   Úforward%   s    
zSwishJitAutoFn.forwardc                 C   s   | j d }t||ƒS ©Nr   )Úsaved_tensorsr   ©r   r   r	   r
   r
   r   Úbackward*   s    
zSwishJitAutoFn.backwardN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ústaticmethodr   r   r   r
   r
   r
   r   r      s   

r   Fc                 C   s
   t  | ¡S r   ©r   Úapply©r	   Úinplacer
   r
   r   Úswish_me0   s    r(   c                       s,   e Zd Zdedœ‡ fdd„Zdd„ Z‡  ZS )ÚSwishMeF©r'   c                    s   t t| ƒ ¡  d S r   )Úsuperr)   Ú__init__©Úselfr'   ©Ú	__class__r
   r   r,   5   s    zSwishMe.__init__c                 C   s
   t  |¡S r   r$   ©r.   r	   r
   r
   r   r   8   s    zSwishMe.forward)F©r   r    r!   Úboolr,   r   Ú__classcell__r
   r
   r/   r   r)   4   s   r)   c                 C   s   |   t t | ¡¡¡S r   )r   r   ÚtanhÚFÚsoftplusr   r
   r
   r   Úmish_jit_fwd<   s    r8   c                 C   s6   t  | ¡}t | ¡ ¡ }| || | d||    ¡S r   )r   r   r6   r7   r5   r   )r	   r   r   Z	x_tanh_spr
   r
   r   Úmish_jit_bwdA   s    
r9   c                   @   s(   e Zd ZdZedd„ ƒZedd„ ƒZdS )ÚMishJitAutoFnzŸ Mish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    A memory efficient, jit scripted variant of Mish
    c                 C   s   |   |¡ t|ƒS r   )r   r8   r   r
   r
   r   r   L   s    
zMishJitAutoFn.forwardc                 C   s   | j d }t||ƒS r   )r   r9   r   r
   r
   r   r   Q   s    
zMishJitAutoFn.backwardN©r   r    r!   r"   r#   r   r   r
   r
   r
   r   r:   H   s
   
r:   c                 C   s
   t  | ¡S r   ©r:   r%   r&   r
   r
   r   Úmish_meW   s    r=   c                       s,   e Zd Zdedœ‡ fdd„Zdd„ Z‡  ZS )ÚMishMeFr*   c                    s   t t| ƒ ¡  d S r   )r+   r>   r,   r-   r/   r
   r   r,   \   s    zMishMe.__init__c                 C   s
   t  |¡S r   r<   r1   r
   r
   r   r   _   s    zMishMe.forward)Fr2   r
   r
   r/   r   r>   [   s   r>   r*   c                 C   s   | d j ddd� d¡S ©Né   r   é   ©ÚminÚmaxç      @©ÚclampÚdivr&   r
   r
   r   Úhard_sigmoid_jit_fwdc   s    rI   c                 C   s&   t  | ¡| dk| dk@  d }|| S )Nç      Àç      @rE   )r   Ú	ones_like©r	   r   Úmr
   r
   r   Úhard_sigmoid_jit_bwdh   s    rO   c                   @   s$   e Zd Zedd„ ƒZedd„ ƒZdS )ÚHardSigmoidJitAutoFnc                 C   s   |   |¡ t|ƒS r   )r   rI   r   r
   r
   r   r   o   s    
zHardSigmoidJitAutoFn.forwardc                 C   s   | j d }t||ƒS r   )r   rO   r   r
   r
   r   r   t   s    
zHardSigmoidJitAutoFn.backwardN)r   r    r!   r#   r   r   r
   r
   r
   r   rP   n   s   
rP   c                 C   s
   t  | ¡S r   ©rP   r%   r&   r
   r
   r   Úhard_sigmoid_mez   s    rR   c                       s,   e Zd Zdedœ‡ fdd„Zdd„ Z‡  ZS )ÚHardSigmoidMeFr*   c                    s   t t| ƒ ¡  d S r   )r+   rS   r,   r-   r/   r
   r   r,      s    zHardSigmoidMe.__init__c                 C   s
   t  |¡S r   rQ   r1   r
   r
   r   r   ‚   s    zHardSigmoidMe.forward)Fr2   r
   r
   r/   r   rS   ~   s   rS   c                 C   s   | | d j ddd� d¡ S r?   rF   r   r
   r
   r   Úhard_swish_jit_fwd†   s    rT   c                 C   s<   t  | ¡| dk }t  | dk| dk@ | d d |¡}|| S )NrK   rJ   ç      à?©r   rL   ÚwhererM   r
   r
   r   Úhard_swish_jit_bwd‹   s    "rX   c                   @   s4   e Zd ZdZedd„ ƒZedd„ ƒZedd„ ƒZdS )	ÚHardSwishJitAutoFnz5A memory efficient, jit-scripted HardSwish activationc                 C   s   |   |¡ t|ƒS r   )r   rT   r   r
   r
   r   r   ”   s    
zHardSwishJitAutoFn.forwardc                 C   s   | j d }t||ƒS r   )r   rX   r   r
   r
   r   r   ™   s    
zHardSwishJitAutoFn.backwardc                 C   s’   |   d|| j dtjdtjd�d�¡}|   d|| j dtjdtjd�d�| j dtjdtjd�d�¡}|   d	|| j dtjdtjd�d�¡}|   d
||¡S )NÚAddÚConstantr@   )Zdtype)Zvalue_tZClipr   rA   ÚDivr   )r   r   ZtensorÚfloat)r   r.   ÚinputZ	hardtanh_r
   r
   r   r   ž   s    $<$zHardSwishJitAutoFn.symbolicN)r   r    r!   r"   r#   r   r   r   r
   r
   r
   r   rY   ’   s   

rY   c                 C   s
   t  | ¡S r   ©rY   r%   r&   r
   r
   r   Úhard_swish_me¦   s    r`   c                       s,   e Zd Zdedœ‡ fdd„Zdd„ Z‡  ZS )ÚHardSwishMeFr*   c                    s   t t| ƒ ¡  d S r   )r+   ra   r,   r-   r/   r
   r   r,   «   s    zHardSwishMe.__init__c                 C   s
   t  |¡S r   r_   r1   r
   r
   r   r   ®   s    zHardSwishMe.forward)Fr2   r
   r
   r/   r   ra   ª   s   ra   c                 C   s   d|  | d j ddd� S )NrU   é   r   rB   )rG   r   r
   r
   r   Úhard_mish_jit_fwd²   s    rc   c                 C   s8   t  | ¡| dk }t  | dk| dk@ | d |¡}|| S )Ng       Àg        g      ð?rV   rM   r
   r
   r   Úhard_mish_jit_bwd·   s    rd   c                   @   s(   e Zd ZdZedd„ ƒZedd„ ƒZdS )ÚHardMishJitAutoFnzã A memory efficient, jit scripted variant of Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    c                 C   s   |   |¡ t|ƒS r   )r   rc   r   r
   r
   r   r   Ã   s    
zHardMishJitAutoFn.forwardc                 C   s   | j d }t||ƒS r   )r   rd   r   r
   r
   r   r   È   s    
zHardMishJitAutoFn.backwardNr;   r
   r
   r
   r   re   ¾   s
   
re   c                 C   s
   t  | ¡S r   ©re   r%   r&   r
   r
   r   Úhard_mish_meÎ   s    rg   c                       s,   e Zd Zdedœ‡ fdd„Zdd„ Z‡  ZS )Ú
HardMishMeFr*   c                    s   t t| ƒ ¡  d S r   )r+   rh   r,   r-   r/   r
   r   r,   Ó   s    zHardMishMe.__init__c                 C   s
   t  |¡S r   rf   r1   r
   r
   r   r   Ö   s    zHardMishMe.forward)Fr2   r
   r
   r/   r   rh   Ò   s   rh   )F)F)F)F)F)F)%r"   r   r   Ztorch.nnr   r6   ZjitÚscriptr   r   ZautogradÚFunctionr   r(   ÚModuler)   r8   r9   r:   r=   r>   r3   rI   rO   rP   rR   rS   rT   rX   rY   r`   ra   rc   rd   re   rg   rh   r
   r
   r
   r   Ú<module>   sL   











