U
    ¯Ê¹j&6  ã                   @   s|  d Z ddlmZmZ ddlZddlmZ ddlm  mZ	 ddl
mZ ddlmZ d/edœd	d
„Zd0edœdd„Zd1edœdd„Zd2eeee f edœdd„Zd3eeedœdd„Zd4eeeedœdd„Zd5eedœdd„ZG dd„ dejƒZG dd „ d ejƒZG d!d"„ d"ejƒZG d#d$„ d$ejƒZG d%d&„ d&eƒZG d'd(„ d(ejƒZG d)d*„ d*eƒZG d+d,„ d,ejƒZ G d-d.„ d.e ƒZ!dS )6a7   EvoNorm in PyTorch

Based on `Evolving Normalization-Activation Layers` - https://arxiv.org/abs/2004.02967
@inproceedings{NEURIPS2020,
 author = {Liu, Hanxiao and Brock, Andy and Simonyan, Karen and Le, Quoc},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
 pages = {13539--13550},
 publisher = {Curran Associates, Inc.},
 title = {Evolving Normalization-Activation Layers},
 url = {https://proceedings.neurips.cc/paper/2020/file/9d4c03631b8b0c85ae08bf05eda37d0f-Paper.pdf},
 volume = {33},
 year = {2020}
}

An attempt at getting decent performing EvoNorms running in PyTorch.
While faster than other PyTorch impl, still quite a ways off the built-in BatchNorm
in terms of memory usage and throughput on GPUs.

I'm testing these modules on TPU w/ PyTorch XLA. Promising start but
currently working around some issues with builtin torch/tensor.var/std. Unlike
GPU, similar train speeds for EvoNormS variants and BatchNorm.

Hacked together by / Copyright 2020 Ross Wightman
é    )ÚSequenceÚUnionNé   )Úcreate_act_layer)Ú_assertçñhãˆµøä>)Úepsc                 C   s2   |   ¡ jdddd� |¡ ¡  | j¡}| | j¡S )N©é   é   FT©ÚdimÚunbiasedÚkeepdim)ÚfloatÚvarÚaddÚsqrtÚtoÚdtypeÚexpandÚshape©Úxr   Ústd© r   úS/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/layers/evo_norm.pyÚinstance_std$   s    &r   c                 C   s"   t | dd� |¡ ¡ }| | j¡S )Nr	   )r   )Ú
manual_varr   r   r   r   r   r   r   r   Úinstance_std_tpu)   s    r   c                 C   s4   |   ¡  ¡ jddd� |¡ ¡  | j¡}| | j¡S )Nr	   T©r   r   )	r   ÚsquareÚmeanr   r   r   r   r   r   )r   r   Úrmsr   r   r   Úinstance_rms/   s    (r$   F©r   Údiff_sqmc                 C   sR   | j |dd�}|r4| |  j |dd�||   d¡}n| | | |  j |dd�}|S )NTr    r   )r"   Úclamp)r   r   r&   Zxmr   r   r   r   r   4   s
    "r   é    )Úgroupsr   Úflattenc           
      C   s°   | j \}}}}| j}t|| dkdƒ |r^|  ||d¡} |  ¡ jdddd� |¡ ¡  |¡}	n:|  |||| ||¡} |  ¡ jdddd� |¡ ¡  |¡}	|	 	| j ¡ ||||¡S )	Nr   Ú éÿÿÿÿr
   FTr   ©r
   r   é   )
r   r   r   Úreshaper   r   r   r   r   r   )
r   r)   r   r*   ÚBÚCÚHÚWÚx_dtyper   r   r   r   Ú	group_std>   s    &$r5   )r)   r   r&   r*   c           
      C   sˆ   | j \}}}}t|| dkdƒ |rB|  ||d¡} t| d|d�}	n$|  |||| ||¡} t| d|d�}	|	 |¡ ¡  | j ¡ ||||¡S )Nr   r+   r,   r%   r-   )r   r   r/   r   r   r   r   )
r   r)   r   r&   r*   r0   r1   r2   r3   r   r   r   r   Úgroup_std_tpuK   s    r6   )r)   r   c           	      C   sz   | j \}}}}t|| dkdƒ | j}|  |||| ||¡} |  ¡  ¡ jddd� |¡ ¡  	|¡}| 
| j ¡ ||||¡S )Nr   r+   r-   Tr    )r   r   r   r/   r   r!   r"   r   Úsqrt_r   r   )	r   r)   r   r0   r1   r2   r3   r4   r#   r   r   r   Ú	group_rmsZ   s    &r8   c                       s.   e Zd Zd
‡ fdd„	Zdd„ Zdd	„ Z‡  ZS )ÚEvoNorm2dB0Tçš™™™™™¹?çü©ñÒMbP?c                    sx   t ƒ  ¡  || _|| _|| _t t |¡¡| _	t t 
|¡¡| _|rTt t |¡¡nd | _|  dt |¡¡ |  ¡  d S ©NÚrunning_var)ÚsuperÚ__init__Ú	apply_actÚmomentumr   ÚnnÚ	ParameterÚtorchÚonesÚweightÚzerosÚbiasÚvÚregister_bufferÚreset_parameters©ÚselfÚnum_featuresr@   rA   r   Ú_©Ú	__class__r   r   r?   d   s    
zEvoNorm2dB0.__init__c                 C   s8   t j | j¡ t j | j¡ | jd k	r4t j | j¡ d S ©N©rB   ÚinitÚones_rF   Úzeros_rH   rI   ©rM   r   r   r   rK   o   s    
zEvoNorm2dB0.reset_parametersc           	      C   s  t | ¡ dkdƒ |j}d}| jd k	rà| jr„| ¡ jddd�}| ¡ |jd  }| j	 
| j	d| j  | ¡ | j ||d    ¡ n| j	}| | j¡ ¡  |¡ |¡ |¡}| j |¡ |¡}|| t|| jƒ }|| |¡ }|| j |¡ |¡ | j |¡ |¡ S ©Nr.   úexpected 4D input©r   r,   r   r   )r   r
   r   F)r   r   r   )r   r   r   rI   Útrainingr   r   Únumelr   r=   Úcopy_rA   Údetachr   r   r7   r   ÚviewZ	expand_asr   ÚmaxrF   rH   )	rM   r   r4   Úv_shaper   ÚnÚleftrI   Úrightr   r   r   Úforwardu   s$    
ÿÿ"zEvoNorm2dB0.forward)Tr:   r;   ©Ú__name__Ú
__module__Ú__qualname__r?   rK   re   Ú__classcell__r   r   rP   r   r9   c   s   r9   c                       s.   e Zd Zd
‡ fdd„	Zdd„ Zdd	„ Z‡  ZS )ÚEvoNorm2dB1Tr:   r   c                    s^   t ƒ  ¡  || _|| _|| _t t |¡¡| _	t t 
|¡¡| _|  dt |¡¡ |  ¡  d S r<   ©r>   r?   r@   rA   r   rB   rC   rD   rE   rF   rG   rH   rJ   rK   rL   rP   r   r   r?   ‹   s    
zEvoNorm2dB1.__init__c                 C   s    t j | j¡ t j | j¡ d S rR   ©rB   rT   rU   rF   rV   rH   rW   r   r   r   rK   •   s    zEvoNorm2dB1.reset_parametersc                 C   sú   t | ¡ dkdƒ |j}d}| jrÒ| jrŠ| ¡ jddd�}| ¡ |jd  }| j	 
| j	d| j  | ¡  | j	j¡| j ||d    ¡ n| j	}| |¡ |¡}| | j¡ ¡ }|d t|| jƒ }|| |¡ }|| j |¡ |¡ | j |¡ |¡ S rX   ©r   r   r   r@   r[   r   r   r\   r   r=   r]   rA   r^   r   r_   r   r   r7   r$   r`   rF   rH   ©rM   r   r4   ra   r   rb   rc   rd   r   r   r   re   ™   s$    "ÿÿzEvoNorm2dB1.forward)Tr:   r   rf   r   r   rP   r   rk   Š   s   
rk   c                       s.   e Zd Zd
‡ fdd„	Zdd„ Zdd	„ Z‡  ZS )ÚEvoNorm2dB2Tr:   r   c                    s^   t ƒ  ¡  || _|| _|| _t t |¡¡| _	t t 
|¡¡| _|  dt |¡¡ |  ¡  d S r<   rl   rL   rP   r   r   r?   ®   s    
zEvoNorm2dB2.__init__c                 C   s    t j | j¡ t j | j¡ d S rR   rm   rW   r   r   r   rK   ¸   s    zEvoNorm2dB2.reset_parametersc                 C   sö   t | ¡ dkdƒ |j}d}| jrÎ| jrŠ| ¡ jddd�}| ¡ |jd  }| j	 
| j	d| j  | ¡  | j	j¡| j ||d    ¡ n| j	}| |¡ |¡}| | j¡ ¡ }t|| jƒ| }|| |¡ }|| j |¡ |¡ | j |¡ |¡ S rX   rn   ro   r   r   r   re   ¼   s$    "ÿÿzEvoNorm2dB2.forward)Tr:   r   rf   r   r   rP   r   rp   ­   s   
rp   c                       s.   e Zd Zd‡ fdd„	Zdd„ Zd	d
„ Z‡  ZS )ÚEvoNorm2dS0r(   NTr   c                    s†   t ƒ  ¡  || _|r0|| dks$t‚|| | _n|| _|| _t t 	|¡¡| _
t t |¡¡| _|rtt t 	|¡¡nd | _|  ¡  d S ©Nr   )r>   r?   r@   ÚAssertionErrorr)   r   rB   rC   rD   rE   rF   rG   rH   rI   rK   ©rM   rN   r)   Ú
group_sizer@   r   rO   rP   r   r   r?   Ñ   s    
zEvoNorm2dS0.__init__c                 C   s8   t j | j¡ t j | j¡ | jd k	r4t j | j¡ d S rR   rS   rW   r   r   r   rK   ß   s    
zEvoNorm2dS0.reset_parametersc                 C   s€   t | ¡ dkdƒ |j}d}| jd k	rX| j |¡ |¡}|||  ¡  t|| j| j	ƒ }|| j
 |¡ |¡ | j |¡ |¡ S ©Nr.   rY   rZ   )r   r   r   rI   r_   r   Úsigmoidr5   r)   r   rF   rH   )rM   r   r4   ra   rI   r   r   r   re   å   s    
 zEvoNorm2dS0.forward)r(   NTr   rf   r   r   rP   r   rq   Ð   s   rq   c                       s&   e Zd Zd	‡ fdd„	Zdd„ Z‡  ZS )
ÚEvoNorm2dS0ar(   NTr;   c                    s   t ƒ j|||||d� d S )N)r)   ru   r@   r   ©r>   r?   rt   rP   r   r   r?   ð   s        ÿzEvoNorm2dS0a.__init__c                 C   sˆ   t | ¡ dkdƒ |j}d}t|| j| jƒ}| jd k	rX| j |¡ |¡}|||  	¡  }|| }|| j
 |¡ |¡ | j |¡ |¡ S rv   )r   r   r   r5   r)   r   rI   r_   r   rw   rF   rH   )rM   r   r4   ra   ÚdrI   r   r   r   re   ô   s    
zEvoNorm2dS0a.forward)r(   NTr;   ©rg   rh   ri   r?   re   rj   r   r   rP   r   rx   ï   s   rx   c                       s.   e Zd Zd‡ fdd„	Zdd„ Zd	d
„ Z‡  ZS )ÚEvoNorm2dS1r(   NTr   c                    sž   t ƒ  ¡  |ptj}|| _|d k	r2|r2t|ƒ| _n
t ¡ | _|r\|| dksPt‚|| | _	n|| _	|| _
d| _t t |¡¡| _t t |¡¡| _|  ¡  d S )Nr   F)r>   r?   rB   ÚSiLUr@   r   ÚactÚIdentityrs   r)   r   Zpre_act_normrC   rD   rE   rF   rG   rH   rK   ©rM   rN   r)   ru   r@   Ú	act_layerr   rO   rP   r   r   r?     s    


zEvoNorm2dS1.__init__c                 C   s    t j | j¡ t j | j¡ d S rR   rm   rW   r   r   r   rK     s    zEvoNorm2dS1.reset_parametersc                 C   sd   t | ¡ dkdƒ |j}d}| jr<|  |¡t|| j| jƒ }|| j 	|¡ 
|¡ | j 	|¡ 
|¡ S rv   )r   r   r   r@   r~   r5   r)   r   rF   r_   r   rH   ©rM   r   r4   ra   r   r   r   re     s    zEvoNorm2dS1.forward)r(   NTNr   rf   r   r   rP   r   r|      s           þr|   c                       s&   e Zd Zd	‡ fdd„	Zdd„ Z‡  ZS )
ÚEvoNorm2dS1ar(   NTr;   c                    s   t ƒ j||||||d� d S ©N)r)   ru   r@   r�   r   ry   r€   rP   r   r   r?   $  s         ÿzEvoNorm2dS1a.__init__c                 C   s^   t | ¡ dkdƒ |j}d}|  |¡t|| j| jƒ }|| j |¡ 	|¡ | j
 |¡ 	|¡ S rv   )r   r   r   r~   r5   r)   r   rF   r_   r   rH   r‚   r   r   r   re   *  s
    zEvoNorm2dS1a.forward)r(   NTNr;   r{   r   r   rP   r   rƒ   #  s           þrƒ   c                       s.   e Zd Zd‡ fdd„	Zdd„ Zd	d
„ Z‡  ZS )ÚEvoNorm2dS2r(   NTr   c                    s˜   t ƒ  ¡  |ptj}|| _|d k	r2|r2t|ƒ| _n
t ¡ | _|r\|| dksPt‚|| | _	n|| _	|| _
t t |¡¡| _t t |¡¡| _|  ¡  d S rr   )r>   r?   rB   r}   r@   r   r~   r   rs   r)   r   rC   rD   rE   rF   rG   rH   rK   r€   rP   r   r   r?   3  s    


zEvoNorm2dS2.__init__c                 C   s    t j | j¡ t j | j¡ d S rR   rm   rW   r   r   r   rK   G  s    zEvoNorm2dS2.reset_parametersc                 C   sd   t | ¡ dkdƒ |j}d}| jr<|  |¡t|| j| jƒ }|| j 	|¡ 
|¡ | j 	|¡ 
|¡ S rv   )r   r   r   r@   r~   r8   r)   r   rF   r_   r   rH   r‚   r   r   r   re   K  s    zEvoNorm2dS2.forward)r(   NTNr   rf   r   r   rP   r   r…   2  s           þr…   c                       s&   e Zd Zd	‡ fdd„	Zdd„ Z‡  ZS )
ÚEvoNorm2dS2ar(   NTr;   c                    s   t ƒ j||||||d� d S r„   ry   r€   rP   r   r   r?   U  s         ÿzEvoNorm2dS2a.__init__c                 C   s^   t | ¡ dkdƒ |j}d}|  |¡t|| j| jƒ }|| j |¡ 	|¡ | j
 |¡ 	|¡ S rv   )r   r   r   r~   r8   r)   r   rF   r_   r   rH   r‚   r   r   r   re   [  s
    zEvoNorm2dS2a.forward)r(   NTNr;   r{   r   r   rP   r   r†   T  s           þr†   )r   )r   )r   )F)r(   r   F)r(   r   FF)r(   r   )"Ú__doc__Útypingr   r   rD   Ztorch.nnrB   Ztorch.nn.functionalZ
functionalÚFZ
create_actr   Ztrace_utilsr   r   r   r   r$   ÚintÚboolr   r5   r6   r8   ÚModuler9   rk   rp   rq   rx   r|   rƒ   r…   r†   r   r   r   r   Ú<module>   s,   
	'###"