U
    ¯Ê¹j
D  ã                   @   s$  d Z ddlmZmZmZmZ ddlZddlmZ ddlm	Z
 ddlmZ ddlmZ dd	lmZmZmZ dd
lmZ d%dd„ZG dd„ dejƒZG dd„ dejƒZd&dd„ZG dd„ dejjƒZdd„ Zdd„ Zdd„ ZG dd„ dej ƒZ!G dd „ d ej ƒZ"G d!d"„ d"ej#ƒZ$G d#d$„ d$ej#ƒZ%dS )'aw   Normalization + Activation Layers

Provides Norm+Act fns for standard PyTorch norm layers such as
* BatchNorm
* GroupNorm
* LayerNorm

This allows swapping with alternative layers that are natively both norm + act such as
* EvoNorm (evo_norm.py)
* FilterResponseNorm (filter_response_norm.py)
* InplaceABN (inplace_abn.py)

Hacked together by / Copyright 2022 Ross Wightman
é    )ÚUnionÚListÚOptionalÚAnyN)Únn)Ú
functional)ÚFrozenBatchNorm2dé   )Úget_act_layer)Úis_fast_normÚfast_group_normÚfast_layer_norm)Ú_assertFTc                 C   s@   t | ƒ} |pi }| d k	r4|r4|r(||d< | f |Ž}nt ¡ }|S )NÚinplace)r
   r   ÚIdentity)Ú	act_layerÚ
act_kwargsr   Ú	apply_actÚact© r   úS/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/layers/norm_act.pyÚ_create_act   s    r   c                       sB   e Zd ZdZdddddejdddddf‡ fdd„	Zdd	„ Z‡  ZS )
ÚBatchNormAct2dzúBatchNorm + Activation

    This module performs BatchNorm + Activation in a manner that will remain backwards
    compatible with weights trained with separate bn, act. This is why we inherit from BN
    instead of composing it as a .bn member.
    çñhãˆµøä>gš™™™™™¹?TNc              
      sŽ   z0||dœ}t t| ƒj|f||||dœ|—Ž W n. tk
r^   t t| ƒj|||||d� Y nX |
d k	rn|
ƒ nt ¡ | _t|||	|d�| _d S )N)ÚdeviceÚdtype)ÚepsÚmomentumÚaffineÚtrack_running_stats©r   r   r   )	Úsuperr   Ú__init__Ú	TypeErrorr   r   Údropr   r   )ÚselfÚnum_featuresr   r   r   r   r   r   r   r   Ú
drop_layerr   r   Zfactory_kwargs©Ú	__class__r   r   r"   .   s,    

ÿûú

ûzBatchNormAct2d.__init__c              
   C   sì   t |jdkd|j› d�ƒ | jd kr*d}n| j}| jrr| jrr| jd k	rr| j d¡ | jd krldt| jƒ }n| j}| jr~d}n| jd ko�| j	d k}t
 || jr¤| jrª| jnd | jr¸| jr¾| j	nd | j| j||| j¡}|  |¡}|  |¡}|S )Né   zexpected 4D input (got zD input)g        r	   g      ð?T)r   Úndimr   Ztrainingr   Únum_batches_trackedZadd_ÚfloatÚrunning_meanÚrunning_varÚFZ
batch_normÚweightÚbiasr   r$   r   )r%   ÚxZexponential_average_factorZbn_trainingr   r   r   ÚforwardS   s4    


÷

zBatchNormAct2d.forward)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚReLUr"   r4   Ú__classcell__r   r   r(   r   r   '   s   	ó%r   c                       s(   e Zd Zejejdœ‡ fdd„Z‡  ZS )ÚSyncBatchNormAct©r3   Úreturnc                    s8   t ƒ  |¡}t| dƒr |  |¡}t| dƒr4|  |¡}|S )Nr$   r   )r!   r4   Úhasattrr$   r   ©r%   r3   r(   r   r   r4   ‹   s    



zSyncBatchNormAct.forward)r5   r6   r7   ÚtorchÚTensorr4   r:   r   r   r(   r   r;   †   s   r;   c              	   C   sî   | }t | tjjjjƒrÄt | tƒrPt| j| j	| j
| j| j|d�}| j|_| j|_n tj | j| j	| j
| j| j|¡}| jršt ¡ � | j|_| j|_W 5 Q R X | j|_| j|_| j|_t| dƒrÄ| j|_|  ¡ D ]\}}| |t||ƒ¡ qÌ~ |S )N)Úprocess_groupÚqconfig)Ú
isinstancer@   r   ÚmodulesÚ	batchnormZ
_BatchNormr   r;   r&   r   r   r   r   r   r$   ÚSyncBatchNormZno_gradr1   r2   r.   r/   r,   r>   rC   Únamed_childrenÚ
add_moduleÚconvert_sync_batchnorm)ÚmodulerB   Zmodule_outputÚnameÚchildr   r   r   rJ   ”   sD    
ú	
ú

rJ   c                       s„   e Zd ZdZddejdddfeedœ‡ fdd„Ze	e
e	eee
 ee
 ee
 dœ‡ fd	d
„Zejejdœdd„Ze
dœdd„Z‡  ZS )ÚFrozenBatchNormAct2da$  
    BatchNormAct2d where the batch statistics and the affine parameters are fixed

    Args:
        num_features (int): Number of features ``C`` from an expected input of size ``(N, C, H, W)``
        eps (float): a value added to the denominator for numerical stability. Default: 1e-5
    r   TN)r&   r   c                    s†   t ƒ  ¡  || _|  dt |¡¡ |  dt |¡¡ |  dt |¡¡ |  dt |¡¡ |d k	rf|ƒ nt ¡ | _	t
||||d�| _d S )Nr1   r2   r.   r/   r    )r!   r"   r   Zregister_bufferr@   ZonesZzerosr   r   r$   r   r   )r%   r&   r   r   r   r   r   r'   r(   r   r   r"   Ç   s    

zFrozenBatchNormAct2d.__init__)Ú
state_dictÚprefixÚlocal_metadataÚstrictÚmissing_keysÚunexpected_keysÚ
error_msgsc           	   	      s2   |d }||kr||= t ƒ  |||||||¡ d S )Nr,   )r!   Ú_load_from_state_dict)	r%   rO   rP   rQ   rR   rS   rT   rU   Znum_batches_tracked_keyr(   r   r   rV   Û   s    
      ÿz*FrozenBatchNormAct2d._load_from_state_dictr<   c                 C   s†   | j  dddd¡}| j dddd¡}| j dddd¡}| j dddd¡}||| j  ¡  }|||  }|| | }|  |  |¡¡}|S )Nr	   éÿÿÿÿ)	r1   Zreshaper2   r/   r.   r   Zrsqrtr   r$   )r%   r3   ÚwÚbÚrvZrmÚscaler2   r   r   r   r4   í   s    zFrozenBatchNormAct2d.forward)r=   c                 C   s,   | j j› d| jjd › d| j› d| j› d�S )Nú(r   z, eps=z, act=ú))r)   r5   r1   Úshaper   r   )r%   r   r   r   Ú__repr__ú   s    zFrozenBatchNormAct2d.__repr__)r5   r6   r7   r8   r   r9   Úintr-   r"   ÚdictÚstrÚboolr   rV   r@   rA   r4   r_   r:   r   r   r(   r   rN   ¾   s*   øýørN   c                 C   sL  | }t | ttfƒrŒt| jƒ}| j|_| j|_| jrZ| jj ¡  	¡ |j_| j
j ¡  	¡ |j
_| jj|j_| jj|j_| j|_| j|_| j|_n¼t | tjjjjtjjjjfƒ�rt| jƒ}| j|_| j|_| jrô| jj ¡  	¡ |j_| j
j ¡  	¡ |j
_| jj|j_| jj|j_| j|_n2|  ¡ D ](\}}t|ƒ}||k	�r| ||¡ �q|S )aÂ  
    Converts all `BatchNorm2d` and `SyncBatchNorm` or `BatchNormAct2d` and `SyncBatchNormAct2d` layers
    of provided module into `FrozenBatchNorm2d` or `FrozenBatchNormAct2d` respectively.

    Args:
        module (torch.nn.Module): Any PyTorch module.

    Returns:
        torch.nn.Module: Resulting module

    Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
    )rD   r   r;   rN   r&   r   r1   ÚdataÚcloneÚdetachr2   r.   r/   r   r$   r   r@   r   rE   rF   ÚBatchNorm2drG   r   rH   Úfreeze_batch_norm_2drI   ©rK   ÚresrL   rM   Ú	new_childr   r   r   rh   þ   s8    

 


rh   c                 C   s  | }t | tƒrxt| jƒ}| jrF| jj ¡  ¡ |j_| j	j ¡  ¡ |j	_| j
j|j
_| jj|j_| j|_| j|_| j|_n–t | tƒràtj | j¡}| jr¾| jj ¡  ¡ |j_| j	j ¡  ¡ |j	_| j
j|j
_| jj|j_| j|_n.|  ¡ D ]$\}}t|ƒ}||k	rè| ||¡ qè|S )a  
    Converts all `FrozenBatchNorm2d` layers of provided module into `BatchNorm2d`. If `module` is itself and instance
    of `FrozenBatchNorm2d`, it is converted into `BatchNorm2d` and returned. Otherwise, the module is walked
    recursively and submodules are converted in place.

    Args:
        module (torch.nn.Module): Any PyTorch module.

    Returns:
        torch.nn.Module: Resulting module

    Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
    )rD   rN   r   r&   r   r1   rd   re   rf   r2   r.   r/   r   r$   r   r   r@   r   rg   rH   Úunfreeze_batch_norm_2drI   ri   r   r   r   rl   *  s0    




rl   c                 C   s    |r| | dkst ‚| | S |S )Nr   )ÚAssertionError)Únum_channelsÚ
num_groupsÚ
group_sizer   r   r   Ú_num_groupsS  s    rq   c                	       s:   e Zd Zdddddejdddf	‡ fdd„	Zdd„ Z‡  ZS )	ÚGroupNormActé    r   TNc                    sV   t t| ƒjt|||ƒ|||d� |
d k	r.|
ƒ nt ¡ | _t|||	|d�| _t	ƒ | _
d S )N©r   r   r    )r!   rr   r"   rq   r   r   r$   r   r   r   Ú
_fast_norm)r%   rn   ro   r   r   rp   r   r   r   r   r'   r(   r   r   r"   \  s    

üzGroupNormAct.__init__c                 C   sR   | j r t|| j| j| j| jƒ}nt || j| j| j| j¡}|  |¡}|  	|¡}|S ©N©
ru   r   ro   r1   r2   r   r0   Z
group_normr$   r   r?   r   r   r   r4   t  s    

zGroupNormAct.forward©r5   r6   r7   r   r9   r"   r4   r:   r   r   r(   r   rr   Z  s   õrr   c                       s6   e Zd Zdddejdddf‡ fdd„	Zdd„ Z‡  ZS )ÚGroupNorm1Actr   TNc	           	         sN   t t| ƒjd|||d� |d k	r&|ƒ nt ¡ | _t||||d�| _tƒ | _	d S )Nr	   rt   r    )
r!   ry   r"   r   r   r$   r   r   r   ru   ©	r%   rn   r   r   r   r   r   r   r'   r(   r   r   r"     s    zGroupNorm1Act.__init__c                 C   sR   | j r t|| j| j| j| jƒ}nt || j| j| j| j¡}|  |¡}|  	|¡}|S rv   rw   r?   r   r   r   r4   �  s    

zGroupNorm1Act.forwardrx   r   r   r(   r   ry   ~  s   ÷ry   c                       sL   e Zd Zdddejdddfeeee ej	f dœ‡ fdd„Z
dd„ Z‡  ZS )	ÚLayerNormActr   TN)Únormalization_shapec	           	         sT   t t| ƒj|||d� |d k	r$|ƒ nt ¡ | _t|ƒ}t||||d�| _t	ƒ | _
d S ©N)r   Zelementwise_affiner    )r!   r{   r"   r   r   r$   r
   r   r   r   ru   )	r%   r|   r   r   r   r   r   r   r'   r(   r   r   r"   ›  s
    zLayerNormAct.__init__c                 C   sR   | j r t|| j| j| j| jƒ}nt || j| j| j| j¡}|  |¡}|  	|¡}|S rv   )
ru   r   Únormalized_shaper1   r2   r   r0   Ú
layer_normr$   r   r?   r   r   r   r4   ­  s    

zLayerNormAct.forward)r5   r6   r7   r   r9   r   r`   r   r@   ÚSizer"   r4   r:   r   r   r(   r   r{   š  s   ÷þr{   c                       s6   e Zd Zdddejdddf‡ fdd„	Zdd„ Z‡  ZS )ÚLayerNormAct2dr   TNc	           	         sL   t t| ƒj|||d� |d k	r$|ƒ nt ¡ | _t||||d�| _tƒ | _	d S r}   )
r!   r�   r"   r   r   r$   r   r   r   ru   rz   r(   r   r   r"   ¸  s    zLayerNormAct2d.__init__c                 C   sr   |  dddd¡}| jr0t|| j| j| j| jƒ}nt || j| j| j| j¡}|  dddd¡}|  	|¡}|  
|¡}|S )Nr   é   é   r	   )Zpermuteru   r   r~   r1   r2   r   r0   r   r$   r   r?   r   r   r   r4   È  s    

zLayerNormAct2d.forwardrx   r   r   r(   r   r�   ·  s   ÷r�   )NFT)N)&r8   Útypingr   r   r   r   r@   r   Ztorch.nnr   r0   Ztorchvision.ops.miscr   Z
create_actr
   Z	fast_normr   r   r   Ztrace_utilsr   r   rg   r   rG   r;   rJ   ÚModulerN   rh   rl   rq   Z	GroupNormrr   ry   Z	LayerNormr{   r�   r   r   r   r   Ú<module>   s(   
_
*@,)$