U
    ¯Ê¹jm[  ã                	   @   s  d Z dgZddlZddlmZ ddlmZ ddlZddlm	Z	 ddl
mZmZ ddlmZmZmZmZ dd	lmZ dd
lmZ ddlmZmZ G dd„ dej	jƒZG dd„ dej	jƒZG dd„ dej	jƒZG dd„ dej	jƒZG dd„ dej	jƒZG dd„ dej	jƒZ G dd„ dej	jƒZ!G dd„ dej	jƒZ"G dd„ dej	jƒZ#G dd„ dej	jƒZ$G d d„ de	jƒZ%d6d"d#„Z&ee&d$d%�e&d$d%�e&d$d%�e&d$d%�e&d$d%�e&d$d%�d&œƒZ'd7d(d)„Z(ed8d*d+„ƒZ)ed9d,d-„ƒZ*ed:d.d/„ƒZ+ed;d0d1„ƒZ,ed<d2d3„ƒZ-ed=d4d5„ƒZ.dS )>zò EfficientViT (by MSRA)

Paper: `EfficientViT: Memory Efficient Vision Transformer with Cascaded Group Attention`
    - https://arxiv.org/abs/2305.07027

Adapted from official impl at https://github.com/microsoft/Cream/tree/main/EfficientViT
ÚEfficientVitMsraé    N)ÚOrderedDict)ÚDict©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚSqueezeExciteÚSelectAdaptivePool2dÚtrunc_normal_Ú_asserté   )Úbuild_model_with_cfg)Úcheckpoint_seq)Úregister_modelÚgenerate_default_cfgsc                       s.   e Zd Zd‡ fdd„	Ze ¡ dd„ ƒZ‡  ZS )ÚConvNormr   r   c	           	   
      s^   t ƒ  ¡  tj|||||||dd�| _t |¡| _tjj 	| jj
|¡ tjj 	| jjd¡ d S )NF©Úbiasr   )ÚsuperÚ__init__ÚnnÚConv2dÚconvZBatchNorm2dÚbnÚtorchÚinitÚ	constant_Úweightr   )	ÚselfZin_chsZout_chsÚksÚstrideÚpadÚdilationÚgroupsÚbn_weight_init©Ú	__class__© ú\/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/efficientvit_msra.pyr      s
    
zConvNorm.__init__c              	   C   sÈ   | j | j }}|j|j|j d  }|j|d d …d d d f  }|j|j|j |j|j d   }tjj	| 
d¡| j j | 
d¡|jdd … | j j| j j| j j| j jd�}|jj |¡ |jj |¡ |S )Nç      à?r   r   é   )r    Úpaddingr"   r#   )r   r   r   Úrunning_varÚepsr   Úrunning_meanr   r   r   Úsizer#   Úshaper    r+   r"   ÚdataÚcopy_)r   Úcr   ÚwÚbÚmr'   r'   r(   Úfuse    s$    ÿ     þzConvNorm.fuse)r   r   r   r   r   r   ©Ú__name__Ú
__module__Ú__qualname__r   r   Úno_gradr7   Ú__classcell__r'   r'   r%   r(   r      s   r   c                       s.   e Zd Zd‡ fdd„	Ze ¡ dd„ ƒZ‡  ZS )	Ú
NormLinearTç{®Gáz”?ç        c                    sf   t ƒ  ¡  t |¡| _t |¡| _tj|||d�| _t	| jj
|d� | jjd k	rbtj | jjd¡ d S )Nr   )Ústdr   )r   r   r   ZBatchNorm1dr   ZDropoutÚdropÚLinearÚlinearr
   r   r   r   r   )r   Zin_featuresZout_featuresr   rA   rB   r%   r'   r(   r   0   s    
zNormLinear.__init__c                 C   sØ   | j | j }}|j|j|j d  }|j| j j| j j |j|j d   }|j|d d d …f  }|jd krz|| jjj }n$|j|d d …d f   d¡| jj }t	j
 | d¡| d¡¡}|jj |¡ |jj |¡ |S )Nr)   éÿÿÿÿr   r   )r   rD   r   r,   r-   r   r.   ÚTÚviewr   r   rC   r/   r1   r2   )r   r   rD   r4   r5   r6   r'   r'   r(   r7   :   s    
ÿÿ
$zNormLinear.fuse)Tr?   r@   r8   r'   r'   r%   r(   r>   /   s   
r>   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚPatchMergingc                    sl   t ƒ  ¡  t|d ƒ}t||dddƒ| _tj ¡ | _t||ddd|d�| _	t
|dƒ| _t||dddƒ| _d S )Né   r   r   é   r*   ©r#   g      Ð?)r   r   Úintr   Úconv1r   r   ÚReLUÚactÚconv2r   ÚseÚconv3)r   ÚdimÚout_dimZhid_dimr%   r'   r(   r   L   s    
zPatchMerging.__init__c                 C   s,   |   |  |  |  |  |  |¡¡¡¡¡¡}|S ©N)rR   rQ   rO   rP   rM   ©r   Úxr'   r'   r(   ÚforwardU   s    (zPatchMerging.forward©r9   r:   r;   r   rX   r=   r'   r'   r%   r(   rH   K   s   	rH   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )ÚResidualDropr@   c                    s   t ƒ  ¡  || _|| _d S rU   )r   r   r6   rB   )r   r6   rB   r%   r'   r(   r   [   s    
zResidualDrop.__init__c              	   C   sd   | j rR| jdkrR||  |¡tj| d¡ddd|jd� | j¡ d| j ¡ 	¡   S ||  |¡ S d S )Nr   r   )Údevice)
ÚtrainingrB   r6   r   Zrandr/   r[   Zge_ÚdivÚdetachrV   r'   r'   r(   rX   `   s        ÿÿÿzResidualDrop.forward)r@   rY   r'   r'   r%   r(   rZ   Z   s   rZ   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚConvMlpc                    s6   t ƒ  ¡  t||ƒ| _tj ¡ | _t||dd�| _d S )Nr   ©r$   )	r   r   r   Úpw1r   r   rN   rO   Úpw2)r   ÚedÚhr%   r'   r(   r   i   s    
zConvMlp.__init__c                 C   s   |   |  |  |¡¡¡}|S rU   )rb   rO   ra   rV   r'   r'   r(   rX   o   s    zConvMlp.forwardrY   r'   r'   r%   r(   r_   h   s   r_   c                       sd   e Zd ZU eeejf ed< d‡ fdd„	Ze 	¡ d‡ fd	d
„	ƒZ
ejejdœdd„Zdd„ Z‡  ZS )ÚCascadedGroupAttentionÚattention_bias_cacheé   rI   é   ©é   rj   rj   rj   c                    s¦  t ƒ  ¡  || _|d | _|| _t|| ƒ| _|| _g }g }t|ƒD ]P}	| 	t
|| | jd | j ƒ¡ | 	t
| j| j||	 d||	 d | jd�¡ qDtj |¡| _tj |¡| _tj tj ¡ t
| j| |dd�¡| _tt t|ƒt|ƒ¡ƒ}
t|
ƒ}i }g }|
D ]^}|
D ]R}t|d |d  ƒt|d |d  ƒf}||k�rNt|ƒ||< | 	|| ¡ �q�qtj t |t|ƒ¡¡| _| jdt |¡ ||¡dd	� i | _d S )
Ng      à¿r*   r   rK   r   r`   Úattention_bias_idxsF)Ú
persistent)r   r   Ú	num_headsÚscaleÚkey_dimrL   Úval_dimÚ
attn_ratioÚrangeÚappendr   r   r   Z
ModuleListÚqkvsÚdwsÚ
SequentialrN   ÚprojÚlistÚ	itertoolsÚproductÚlenÚabsÚ	ParameterZzerosÚattention_biasesZregister_bufferZ
LongTensorrG   rf   )r   rS   ro   rm   rq   Ú
resolutionÚkernelsrt   ru   ÚiZpointsÚNZattention_offsetsZidxsÚp1Zp2Úoffsetr%   r'   r(   r   �   s<    	

 .þ(
zCascadedGroupAttention.__init__Tc                    s    t ƒ  |¡ |r| jri | _d S rU   )r   Útrainrf   )r   Úmoder%   r'   r(   r…   «   s    
zCascadedGroupAttention.train)r[   Úreturnc                 C   s^   t j ¡ s| jr$| jd d …| jf S t|ƒ}|| jkrP| jd d …| jf | j|< | j| S d S rU   )r   ÚjitÚ
is_tracingr\   r~   rk   Ústrrf   )r   r[   Z
device_keyr'   r'   r(   Úget_attention_biases±   s    
z+CascadedGroupAttention.get_attention_biasesc                 C   s@  |j \}}}}|jt| jƒdd�}g }|d }|  |j¡}	tt| j| jƒƒD ]Ü\}
\}}|
dkrl|||
  }||ƒ}| 	|d||¡j
| j| j| jgdd�\}}}||ƒ}| d¡| d¡| d¡  }}}|| j }| dd¡| }||	|
  }|jdd�}|| dd¡ }| 	|| j||¡}| |¡ qL|  t |d¡¡}|S )Nr   )rS   r   rE   r*   éþÿÿÿ)r0   Úchunkr{   rt   r‹   r[   Ú	enumerateÚzipru   rG   Úsplitro   rp   Úflattenrn   Ú	transposeZsoftmaxrs   rw   r   Úcat)r   rW   ÚBÚCÚHÚWZfeats_inZ	feats_outZfeatZ	attn_biasZhead_idxZqkvru   ÚqÚkÚvÚattnr'   r'   r(   rX   º   s*    ,"
zCascadedGroupAttention.forward)rg   rI   rh   ri   )T)r9   r:   r;   r   rŠ   r   ZTensorÚ__annotations__r   r<   r…   r[   r‹   rX   r=   r'   r'   r%   r(   re   t   s   
    ù*	re   c                       s*   e Zd ZdZd‡ fdd„	Zd	d
„ Z‡  ZS )ÚLocalWindowAttentiona¹   Local Window Attention.

    Args:
        dim (int): Number of input channels.
        key_dim (int): The dimension for query and key.
        num_heads (int): Number of attention heads.
        attn_ratio (int): Multiplier for the query dim for value dimension.
        resolution (int): Input resolution.
        window_resolution (int): Local window resolution.
        kernels (List[int]): The kernel size of the dw conv on query.
    rg   rI   rh   é   ri   c                    sV   t ƒ  ¡  || _|| _|| _|dks,tdƒ‚|| _t||ƒ}t||||||d�| _	d S )Nr   z"window_size must be greater than 0)rq   r   r€   )
r   r   rS   rm   r   ÚAssertionErrorÚwindow_resolutionÚminre   r›   ©r   rS   ro   rm   rq   r   r    r€   r%   r'   r(   r   Þ   s    


  üzLocalWindowAttention.__init__c              	   C   s¼  | j  }}|j\}}}}t||kd||f› d||f› �ƒ t||kd||f› d||f› �ƒ || jkr~|| jkr~|  |¡}�n:| dddd¡}| j|| j  | j }| j|| j  | j }	tjj 	|ddd|	d|f¡}|| ||	  }
}|
| j }|| j }| 
||| j|| j|¡ dd¡}| || | | j| j|¡ dddd¡}|  |¡}| dddd¡ 
|||| j| j|¡}| dd¡ ||
||¡}|d d …d |…d |…f  ¡ }| dddd¡}|S )Nz%input feature has wrong size, expect z, got r   r*   rJ   r   )r   r0   r   r    r›   Zpermuter   r   Z
functionalr!   rG   r’   ZreshapeÚ
contiguous)r   rW   r–   r—   r”   r•   ZH_ZW_Zpad_bZpad_rZpHZpWZnHZnWr'   r'   r(   rX   ö   s*    
""

 (
$zLocalWindowAttention.forward)rg   rI   rh   rž   ri   ©r9   r:   r;   Ú__doc__r   rX   r=   r'   r'   r%   r(   r�   Ò   s        ør�   c                       s<   e Zd ZdZddddddddgf‡ fdd„	Zd	d
„ Z‡  ZS )ÚEfficientVitBlockaÕ   A basic EfficientVit building block.

    Args:
        dim (int): Number of input channels.
        key_dim (int): Dimension for query and key in the token mixer.
        num_heads (int): Number of attention heads.
        attn_ratio (int): Multiplier for the query dim for value dimension.
        resolution (int): Input resolution.
        window_resolution (int): Local window resolution.
        kernels (List[int]): The kernel size of the dw conv on query.
    rg   rI   rh   rž   rj   c              
      s’   t ƒ  ¡  tt||ddd|dd�ƒ| _tt|t|d ƒƒƒ| _tt|||||||d�ƒ| _	tt||ddd|dd�ƒ| _
tt|t|d ƒƒƒ| _d S )NrJ   r   r@   )r#   r$   r*   )rq   r   r    r€   )r   r   rZ   r   Údw0r_   rL   Úffn0r�   ÚmixerÚdw1Úffn1r¢   r%   r'   r(   r     s     

  ûÿ
zEfficientVitBlock.__init__c                 C   s"   |   |  |  |  |  |¡¡¡¡¡S rU   )r«   rª   r©   r¨   r§   rV   r'   r'   r(   rX   ;  s    zEfficientVitBlock.forwardr¤   r'   r'   r%   r(   r¦     s   
ør¦   c                	       s<   e Zd Zdddddddddgdf‡ fdd	„	Zd
d„ Z‡  ZS )ÚEfficientVitStage©Ú r   rg   rI   rh   rž   rj   r   c                    s,  t ƒ  ¡  |d dkrÒ|d |d  d | _g }| dtj tt||ddd|d�ƒtt	|t
|d ƒƒƒ¡f¡ | dt||ƒf¡ | d	tj tt||ddd|d�ƒtt	|t
|d ƒƒƒ¡f¡ t t|ƒ¡| _n||ksÞt‚t ¡ | _|| _g }t|
ƒD ] }| t||||| j||	ƒ¡ qútj|Ž | _d S )
Nr   Ú	subsampler   Zres1rJ   rK   r*   Z
patchmergeZres2)r   r   r   rs   r   r   rv   rZ   r   r_   rL   rH   r   Ú
downsamplerŸ   ÚIdentityrr   r¦   Úblocks)r   Úin_dimrT   ro   r°   rm   rq   r   r    r€   ÚdepthZdown_blocksr²   Údr%   r'   r(   r   @  s6    
þþþþ
zEfficientVitStage.__init__c                 C   s   |   |¡}|  |¡}|S rU   )r°   r²   rV   r'   r'   r(   rX   k  s    

zEfficientVitStage.forwardrY   r'   r'   r%   r(   r¬   ?  s   
õ+r¬   c                       s   e Zd Z‡ fdd„Z‡  ZS )ÚPatchEmbeddingc              	      sÂ   t ƒ  ¡  |  dt||d dddƒ¡ |  dtj ¡ ¡ |  dt|d |d dddƒ¡ |  d	tj ¡ ¡ |  d
t|d |d dddƒ¡ |  dtj ¡ ¡ |  dt|d |dddƒ¡ d| _d S )NrM   rg   rJ   r*   r   Zrelu1rP   rI   Zrelu2rR   Zrelu3Zconv4é   )r   r   Z
add_moduler   r   r   rN   Ú
patch_size)r   Úin_chansrS   r%   r'   r(   r   r  s    
  zPatchEmbedding.__init__)r9   r:   r;   r   r=   r'   r'   r%   r(   r¶   q  s   r¶   c                       sŒ   e Zd Zd#‡ fdd„	Zejjdd„ ƒZejjd$dd„ƒZejjd%dd„ƒZ	ejjdd„ ƒZ
d&dd„Zdd„ Zd'edœdd „Zd!d"„ Z‡  ZS )(r   éà   rJ   éè  ©é@   é€   éÀ   ©r·   r·   r·   ©r   r*   rJ   ©rI   rI   rI   ©rž   rž   rž   ri   ©r­   ©r¯   r*   rÅ   Úavgr@   c                    sˆ  t t| ƒ ¡  d| _|| _|| _t|ˆ d ƒ| _| jj}|| jj }‡ ‡‡fdd„t	t
ˆ ƒƒD ƒ}g | _g }ˆ d }ttˆ ˆ|ˆ|||
ƒƒD ]†\}\}}}}}}}t|||||||||	|d�
}|}|d dkrâ|dkrâ||d 9 }|j}| |¡ |  jt||d|› �d	�g7  _qŒtj|Ž | _|d
k�r:t|dd�| _n|dk�sHt‚t ¡ | _ˆ d | _|dk�rxt| j|| jd�ntj ¡ | _d S )NFr   c                    s$   g | ]}ˆ | ˆ| ˆ|   ‘qS r'   r'   )Ú.0r�   ©Ú	embed_dimro   rm   r'   r(   Ú
<listcomp>—  s     z-EfficientVitMsra.__init__.<locals>.<listcomp>)
r³   rT   ro   r°   rm   rq   r   r    r€   r´   r¯   r   zstages.)Znum_chsZ	reductionÚmodulerÆ   T©Z	pool_typer‘   rE   ©rB   )r   r   r   Úgrad_checkpointingÚnum_classesÚ	drop_rater¶   Úpatch_embedr¸   rr   r{   Zfeature_inforŽ   r�   r¬   r   rs   Údictr   rv   Ústagesr	   Úglobal_poolrŸ   r±   Únum_featuresr>   r   Úhead)r   Úimg_sizer¹   rÏ   rÉ   ro   r´   rm   Úwindow_sizer€   Zdown_opsrÔ   rÐ   r    r   rq   rÓ   Zpre_edr�   rc   ÚkdZdpthZnhÚarÚwdÚdoZstager%   rÈ   r(   r     s\    ÿö
"


ÿ  ÿÿzEfficientVitMsra.__init__c                 C   s   dd„ |   ¡  ¡ D ƒS )Nc                 S   s   h | ]}d |kr|’qS )r~   r'   )rÇ   rW   r'   r'   r(   Ú	<setcomp>¾  s      z3EfficientVitMsra.no_weight_decay.<locals>.<setcomp>)Z
state_dictÚkeys©r   r'   r'   r(   Úno_weight_decay¼  s    z EfficientVitMsra.no_weight_decayFc                 C   s   t d|rdnddgd�}|S )Nz^patch_embedz^stages\.(\d+))z^stages\.(\d+).downsample)r   )z^stages\.(\d+)\.\w+\.(\d+)N)Ústemr²   )rÒ   )r   ZcoarseZmatcherr'   r'   r(   Úgroup_matcherÀ  s    þþzEfficientVitMsra.group_matcherTc                 C   s
   || _ d S rU   )rÎ   )r   Úenabler'   r'   r(   Úset_grad_checkpointingË  s    z'EfficientVitMsra.set_grad_checkpointingc                 C   s   | j jS rU   )rÖ   rD   rß   r'   r'   r(   Úget_classifierÏ  s    zEfficientVitMsra.get_classifierNc                 C   sf   || _ |d k	r<|dkr&t|dd�| _n|dks2t‚t ¡ | _|dkrVt| j|| jd�nt	j ¡ | _
d S )NrÆ   TrÌ   r   rÍ   )rÏ   r	   rÔ   rŸ   r   r±   r>   rÕ   rÐ   r   rÖ   )r   rÏ   rÔ   r'   r'   r(   Úreset_classifierÓ  s    
ÿ  ÿÿz!EfficientVitMsra.reset_classifierc                 C   s6   |   |¡}| jr(tj ¡ s(t| j|ƒ}n
|  |¡}|S rU   )rÑ   rÎ   r   rˆ   Zis_scriptingr   rÓ   rV   r'   r'   r(   Úforward_featuresÞ  s
    

z!EfficientVitMsra.forward_features)Ú
pre_logitsc                 C   s   |   |¡}|r|S |  |¡S rU   )rÔ   rÖ   )r   rW   rè   r'   r'   r(   Úforward_headæ  s    
zEfficientVitMsra.forward_headc                 C   s   |   |¡}|  |¡}|S rU   )rç   ré   rV   r'   r'   r(   rX   ê  s    

zEfficientVitMsra.forward)rº   rJ   r»   r¼   rÀ   rÁ   rÂ   rÃ   ri   rÄ   rÆ   r@   )F)T)N)F)r9   r:   r;   r   r   rˆ   Úignorerà   râ   rä   rå   ræ   rç   Úboolré   rX   r=   r'   r'   r%   r(   r   ~  s2               ó=



r®   c              	   K   s   | dt tdddddœ|–S )Nr»   zpatch_embed.conv1.convzhead.linearT)rI   rI   )ÚurlrÏ   ZmeanrA   Z
first_convÚ
classifierZfixed_input_sizeZ	pool_sizer   )rì   Úkwargsr'   r'   r(   Ú_cfg  s    ø	÷rï   ztimm/)Z	hf_hub_id)zefficientvit_m0.r224_in1kzefficientvit_m1.r224_in1kzefficientvit_m2.r224_in1kzefficientvit_m3.r224_in1kzefficientvit_m4.r224_in1kzefficientvit_m5.r224_in1kFc                 K   s0   |  dd¡}tt| |fdtd|d�i|—Ž}|S )NÚout_indices)r   r   r*   Zfeature_cfgT)Zflatten_sequentialrð   )Úpopr   r   rÒ   )ÚvariantÚ
pretrainedrî   rð   Úmodelr'   r'   r(   Ú_create_efficientvit_msra<  s    ý
üûrõ   c              
   K   sN   t ddddgdddgdddgd	d	d	gd
d
d
d
gd�}tdd| it |f|Ž—ŽS )Nrº   r½   r¾   r¿   r   r*   rJ   rI   rž   rj   ©r×   rÉ   r´   rm   rØ   r€   Úefficientvit_m0ró   )r÷   ©rÒ   rõ   ©ró   rî   Z
model_argsr'   r'   r(   r÷   H  s    
úr÷   c              
   K   sN   t ddddgdddgdddgdddgdd	ddgd
�}tdd| it |f|Ž—ŽS )Nrº   r¾   é�   r¿   r   r*   rJ   rž   rj   rö   Úefficientvit_m1ró   )rû   rø   rù   r'   r'   r(   rû   U  s    
úrû   c              
   K   sN   t ddddgdddgdddgdddgdd	ddgd
�}tdd| it |f|Ž—ŽS )Nrº   r¾   r¿   r   r*   rJ   rI   rž   rj   rö   Úefficientvit_m2ró   )rü   rø   rù   r'   r'   r(   rü   b  s    
úrü   c              
   K   sN   t ddddgdddgdddgd	d	d	gd
d
d
d
gd�}tdd| it |f|Ž—ŽS )Nrº   r¾   éð   i@  r   r*   rJ   rI   rž   rj   rö   Úefficientvit_m3ró   )rþ   rø   rù   r'   r'   r(   rþ   o  s    
úrþ   c              
   K   sN   t ddddgdddgdddgd	d	d	gd	d
ddgd�}tdd| it |f|Ž—ŽS )Nrº   r¾   é   é€  r   r*   rJ   rI   rž   rj   rö   Úefficientvit_m4ró   )r  rø   rù   r'   r'   r(   r  |  s    
úr  c              
   K   sN   t ddddgdddgdddgdddgdd	ddgd
�}tdd| it |f|Ž—ŽS )Nrº   r¿   i   r   r   rJ   rI   rž   rj   rö   Úefficientvit_m5ró   )r  rø   rù   r'   r'   r(   r  ‰  s    
úr  )r®   )F)F)F)F)F)F)F)/r¥   Ú__all__ry   Úcollectionsr   Útypingr   r   Ztorch.nnr   Z	timm.datar   r   Ztimm.layersr   r	   r
   r   Z_builderr   Z_manipulater   Ú	_registryr   r   rv   r   r>   ÚModulerH   rZ   r_   re   r�   r¦   r¬   r¶   r   rï   Zdefault_cfgsrõ   r÷   rû   rü   rþ   r  r  r'   r'   r'   r(   Ú<module>   sr   ^A,2 
ÿÿÿÿÿÿë
