U
    ¯Ê¹jPJ  ã                   @   sî  d Z ddlmZ ddlZddlmZ ddlmZmZ ddl	m
Z
mZmZmZmZ ddlmZ ddlmZ dd	lmZmZ d
gZddddœZddddœZG dd„ dejjƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZ G dd„ dejƒZ!G d d!„ d!ejƒZ"G d"d#„ d#ejƒZ#G d$d%„ d%ejƒZ$G d&d'„ d'ejƒZ%G d(d
„ d
ejƒZ&d)d*„ Z'd;d,d-„Z(ee(d.d/�e(d.d/�e(d.d/�d0œƒZ)d<d2d3„Z*ed=e&d4œd5d6„ƒZ+ed>e&d4œd7d8„ƒZ,ed?e&d4œd9d:„ƒZ-dS )@aû   EfficientFormer

@article{li2022efficientformer,
  title={EfficientFormer: Vision Transformers at MobileNet Speed},
  author={Li, Yanyu and Yuan, Geng and Wen, Yang and Hu, Eric and Evangelidis, Georgios and Tulyakov,
   Sergey and Wang, Yanzhi and Ren, Jian},
  journal={arXiv preprint arXiv:2206.01191},
  year={2022}
}

Based on Apache 2.0 licensed code at https://github.com/snap-research/EfficientFormer, Copyright (c) 2022 Snap Inc.

Modifications and timm support by / Copyright 2022, Ross Wightman
é    )ÚDictN©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚtrunc_normal_Ú	to_2tupleÚMlpÚndgridé   )Úbuild_model_with_cfg)Úcheckpoint_seq)Úgenerate_default_cfgsÚregister_modelÚEfficientFormer)é0   é`   éà   iÀ  )é@   é€   i@  i   )r   éÀ   é€  i   )Úl1Úl3Úl7)é   é   é   é   )r   r   é   r   )r   r   é   é   c                       sd   e Zd ZU eeejf ed< d‡ fdd„	Ze 	¡ d‡ fd
d„	ƒZ
ejejdœdd„Zdd„ Z‡  ZS )Ú	AttentionÚattention_bias_cacher   é    r!   r   é   c                    s  t ƒ  ¡  || _|d | _|| _|| | _t|| ƒ| _| j| | _|| _	t
 || jd | j ¡| _t
 | j|¡| _t|ƒ}t tt |d ¡t |d ¡ƒ¡ d¡}|dd d …d f |dd d d …f   ¡ }|d |d  |d  }tj
 t ||d |d  ¡¡| _|  d|¡ i | _d S )Ng      à¿r   r   r   .Úattention_bias_idxs)ÚsuperÚ__init__Ú	num_headsÚscaleÚkey_dimZkey_attn_dimÚintÚval_dimÚval_attn_dimÚ
attn_ratioÚnnÚLinearÚqkvÚprojr   ÚtorchÚstackr
   ZarangeÚflattenÚabsÚ	ParameterZzerosÚattention_biasesZregister_bufferr#   )ÚselfÚdimr+   r)   r/   Ú
resolutionÚposZrel_pos©Ú	__class__© úZ/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/efficientformer.pyr(   -   s"    


*("zAttention.__init__Tc                    s    t ƒ  |¡ |r| jri | _d S ©N)r'   Útrainr#   )r:   Úmoder>   r@   rA   rC   I   s    
zAttention.train)ÚdeviceÚreturnc                 C   s^   t j ¡ s| jr$| jd d …| jf S t|ƒ}|| jkrP| jd d …| jf | j|< | j| S d S rB   )r4   ÚjitÚ
is_tracingÚtrainingr9   r&   Ústrr#   )r:   rE   Z
device_keyr@   r@   rA   Úget_attention_biasesO   s    
zAttention.get_attention_biasesc           
      C   s°   |j \}}}|  |¡}| ||| jd¡ dddd¡}|j| j| j| jgdd�\}}}|| dd¡ | j	 }	|	|  
|j¡ }	|	jdd�}	|	|  dd¡ ||| j¡}|  |¡}|S )Néÿÿÿÿr   r   r   r   ©r;   éþÿÿÿ)Úshaper2   Zreshaper)   ZpermuteÚsplitr+   r-   Ú	transposer*   rK   rE   Zsoftmaxr.   r3   )
r:   ÚxÚBÚNÚCr2   ÚqÚkÚvZattnr@   r@   rA   ÚforwardX   s    
 
zAttention.forward)r   r$   r!   r   r%   )T)Ú__name__Ú
__module__Ú__qualname__r   rJ   r4   ZTensorÚ__annotations__r(   Zno_gradrC   rE   rK   rY   Ú__classcell__r@   r@   r>   rA   r"   *   s   
     ú	r"   c                       s&   e Zd Zejejf‡ fdd„	Z‡  ZS )ÚStem4c              
      s”   t ƒ  ¡  d| _|  dtj||d dddd�¡ |  d||d ƒ¡ |  d|ƒ ¡ |  d	tj|d |dddd�¡ |  d
||ƒ¡ |  d|ƒ ¡ d S )Nr   Zconv1r   r   r   ©Úkernel_sizeÚstrideÚpaddingÚnorm1Zact1Zconv2Únorm2Zact2)r'   r(   rb   Z
add_moduler0   ÚConv2d)r:   Úin_chsÚout_chsÚ	act_layerÚ
norm_layerr>   r@   rA   r(   h   s    
  zStem4.__init__)rZ   r[   r\   r0   ZReLUÚBatchNorm2dr(   r^   r@   r@   r>   rA   r_   g   s   r_   c                       s4   e Zd ZdZdddejf‡ fdd„	Zdd„ Z‡  ZS )	Ú
DownsamplezŽ
    Downsampling via strided conv w/ norm
    Input: tensor in shape [B, C, H, W]
    Output: tensor in shape [B, C, H/stride, W/stride]
    r   r   Nc                    s>   t ƒ  ¡  |d kr|d }tj|||||d�| _||ƒ| _d S )Nr   r`   )r'   r(   r0   rf   ÚconvÚnorm)r:   rg   rh   ra   rb   rc   rj   r>   r@   rA   r(   {   s
    
zDownsample.__init__c                 C   s   |   |¡}|  |¡}|S rB   )rm   rn   ©r:   rR   r@   r@   rA   rY   ‚   s    

zDownsample.forward)	rZ   r[   r\   Ú__doc__r0   rk   r(   rY   r^   r@   r@   r>   rA   rl   t   s   rl   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚFlatc                    s   t ƒ  ¡  d S rB   )r'   r(   ©r:   r>   r@   rA   r(   Š   s    zFlat.__init__c                 C   s   |  d¡ dd¡}|S )Nr   r   )r6   rQ   ro   r@   r@   rA   rY   �   s    zFlat.forward©rZ   r[   r\   r(   rY   r^   r@   r@   r>   rA   rq   ˆ   s   rq   c                       s*   e Zd ZdZd‡ fdd„	Zdd„ Z‡  ZS )ÚPoolingzP
    Implementation of pooling for PoolFormer
    --pool_size: pooling size
    r   c                    s&   t ƒ  ¡  tj|d|d dd�| _d S )Nr   r   F)rb   rc   Zcount_include_pad)r'   r(   r0   Z	AvgPool2dÚpool)r:   Ú	pool_sizer>   r@   rA   r(   ˜   s    
zPooling.__init__c                 C   s   |   |¡| S rB   )ru   ro   r@   r@   rA   rY   œ   s    zPooling.forward)r   )rZ   r[   r\   rp   r(   rY   r^   r@   r@   r>   rA   rt   ’   s   rt   c                       s8   e Zd ZdZddejejdf‡ fdd„	Zdd„ Z‡  Z	S )ÚConvMlpWithNormz`
    Implementation of MLP with 1*1 convolutions.
    Input: tensor with shape [B, C, H, W]
    Nç        c                    s†   t ƒ  ¡  |p|}|p|}t ||d¡| _|d k	r:||ƒnt ¡ | _|ƒ | _t ||d¡| _|d k	rl||ƒnt ¡ | _	t 
|¡| _d S )Nr   )r'   r(   r0   rf   Úfc1ÚIdentityrd   ÚactÚfc2re   ÚDropoutÚdrop)r:   Úin_featuresÚhidden_featuresZout_featuresri   rj   r~   r>   r@   rA   r(   ¦   s    	
zConvMlpWithNorm.__init__c                 C   sJ   |   |¡}|  |¡}|  |¡}|  |¡}|  |¡}|  |¡}|  |¡}|S rB   )ry   rd   r{   r~   r|   re   ro   r@   r@   rA   rY   ¹   s    






zConvMlpWithNorm.forward)
rZ   r[   r\   rp   r0   ÚGELUrk   r(   rY   r^   r@   r@   r>   rA   rw       s   ùrw   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )Ú
LayerScaleçñhãˆµøä>Fc                    s*   t ƒ  ¡  || _t |t |¡ ¡| _d S rB   ©r'   r(   Úinplacer0   r8   r4   ZonesÚgamma©r:   r;   Zinit_valuesr…   r>   r@   rA   r(   Å   s    
zLayerScale.__init__c                 C   s   | j r| | j¡S || j S rB   )r…   Úmul_r†   ro   r@   r@   rA   rY   Ê   s    zLayerScale.forward)rƒ   Frs   r@   r@   r>   rA   r‚   Ä   s   r‚   c                       s6   e Zd Zdejejdddf‡ fdd„	Zdd„ Z‡  ZS )ÚMetaBlock1dç      @rx   rƒ   c                    sx   t ƒ  ¡  ||ƒ| _t|ƒ| _||ƒ| _t|t|| ƒ||d�| _|dkrRt	|ƒnt
 ¡ | _t||ƒ| _t||ƒ| _d S )N)r   r€   ri   r~   rx   )r'   r(   rd   r"   Útoken_mixerre   r	   r,   Úmlpr   r0   rz   Ú	drop_pathr‚   Úls1Úls2)r:   r;   Ú	mlp_ratiori   rj   Ú	proj_dropr�   Úlayer_scale_init_valuer>   r@   rA   r(   Ð   s    





üzMetaBlock1d.__init__c              
   C   sD   ||   |  |  |  |¡¡¡¡ }||   |  |  |  |¡¡¡¡ }|S rB   )r�   rŽ   r‹   rd   r�   rŒ   re   ro   r@   r@   rA   rY   é   s      zMetaBlock1d.forward)	rZ   r[   r\   r0   r�   Ú	LayerNormr(   rY   r^   r@   r@   r>   rA   r‰   Î   s   ør‰   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )ÚLayerScale2drƒ   Fc                    s*   t ƒ  ¡  || _t |t |¡ ¡| _d S rB   r„   r‡   r>   r@   rA   r(   ð   s    
zLayerScale2d.__init__c                 C   s*   | j  dddd¡}| jr"| |¡S || S )Nr   rL   )r†   Úviewr…   rˆ   )r:   rR   r†   r@   r@   rA   rY   õ   s    zLayerScale2d.forward)rƒ   Frs   r@   r@   r>   rA   r”   ï   s   r”   c                       s8   e Zd Zddejejdddf‡ fdd„	Zdd„ Z‡  ZS )	ÚMetaBlock2dr   rŠ   rx   rƒ   c	           	         s‚   t ƒ  ¡  t|d�| _t||ƒ| _|dkr2t|ƒnt ¡ | _	t
|t|| ƒ|||d�| _t||ƒ| _|dkrtt|ƒnt ¡ | _d S )N)rv   rx   )r€   ri   rj   r~   )r'   r(   rt   r‹   r”   rŽ   r   r0   rz   Ú
drop_path1rw   r,   rŒ   r�   Ú
drop_path2)	r:   r;   rv   r�   ri   rj   r‘   r�   r’   r>   r@   rA   r(   ü   s    

ûzMetaBlock2d.__init__c                 C   s8   ||   |  |  |¡¡¡ }||  |  |  |¡¡¡ }|S rB   )r—   rŽ   r‹   r˜   r�   rŒ   ro   r@   r@   rA   rY     s    zMetaBlock2d.forward)	rZ   r[   r\   r0   r�   rk   r(   rY   r^   r@   r@   r>   rA   r–   ú   s   ÷r–   c                
       s@   e Zd Zddddejejejdddf
‡ fdd„	Zd	d
„ Z‡  Z	S )ÚEfficientFormerStageTr   r   rŠ   rx   rƒ   c                    sî   t ƒ  ¡  d| _|r*t|||	d�| _|}n||ks6t‚t ¡ | _g }|r\||kr\| t	ƒ ¡ t
|ƒD ]x}|| d }|r¢||kr¢| t||||
||| |d�¡ qd| t|||||	||| |d�¡ |rd||krd| t	ƒ ¡ qdtj|Ž | _d S )NF)rg   rh   rj   r   )r�   ri   rj   r‘   r�   r’   )rv   r�   ri   rj   r‘   r�   r’   )r'   r(   Úgrad_checkpointingrl   Ú
downsampleÚAssertionErrorr0   rz   Úappendrq   Úranger‰   r–   Ú
SequentialÚblocks)r:   r;   Zdim_outÚdepthr›   Únum_vitrv   r�   ri   rj   Únorm_layer_clr‘   r�   r’   r    Z	block_idxZ
remain_idxr>   r@   rA   r(     sN    

ùÿøÿzEfficientFormerStage.__init__c                 C   s6   |   |¡}| jr(tj ¡ s(t| j|ƒ}n
|  |¡}|S rB   )r›   rš   r4   rG   Úis_scriptingr   r    ro   r@   r@   rA   rY   Z  s
    

zEfficientFormerStage.forward)
rZ   r[   r\   r0   r�   rk   r“   r(   rY   r^   r@   r@   r>   rA   r™     s   ò<r™   c                       sÊ   e Zd Zdddddddddejejejdddf‡ fd	d
„	Zdd„ Ze	j
jdd„ ƒZe	j
jd"dd„ƒZe	j
jd#dd„ƒZe	j
jdd„ ƒZd$dd„Ze	j
jd%dd„ƒZdd„ Zd&edœdd„Zd d!„ Z‡  ZS )'r   Nr   éè  Úavgr   r   rƒ   rx   c                    s\  t ƒ  ¡  || _|| _t||d |d�| _|d }dd„ t d|t|ƒ¡ 	|¡D ƒ}|pjddt
|ƒd   }g }tt
|ƒƒD ]T}t||| || || |dkr¢|nd|	||||||| |
d	�}|| }| |¡ q|tj|Ž | _|d
 | _|| jƒ| _t |¡| _|dk�rt | j|¡nt ¡ | _|dk�r<t |d
 |¡nt ¡ | _d| _|  | j¡ d S )Nr   )rj   c                 S   s   g | ]}|  ¡ ‘qS r@   )Útolist)Ú.0rR   r@   r@   rA   Ú
<listcomp>�  s     z,EfficientFormer.__init__.<locals>.<listcomp>)F)Tr   r   )
r›   r¢   rv   r�   ri   r£   rj   r‘   r�   r’   rL   F)r'   r(   Únum_classesÚglobal_poolr_   Ústemr4   ZlinspaceÚsumrP   Úlenrž   r™   r�   r0   rŸ   ÚstagesÚnum_featuresrn   r}   Ú	head_dropr1   rz   ÚheadÚ	head_distÚdistilled_trainingÚapplyÚ_init_weights)r:   ÚdepthsÚ
embed_dimsZin_chansrª   r«   Zdownsamplesr¢   Z
mlp_ratiosrv   r’   ri   rj   r£   Z	drop_rateZproj_drop_rateZdrop_path_rateÚkwargsZprev_dimZdprr¯   ÚiZstager>   r@   rA   r(   e  sD    
"ó
"$zEfficientFormer.__init__c                 C   sD   t |tjƒr@t|jdd� t |tjƒr@|jd k	r@tj |jd¡ d S )Ng{®Gáz”?)Ústdr   )Ú
isinstancer0   r1   r   ÚweightZbiasÚinitZ	constant_)r:   Úmr@   r@   rA   r¶   ¥  s    zEfficientFormer._init_weightsc                 C   s   dd„ |   ¡ D ƒS )Nc                 S   s   h | ]\}}d |kr|’qS )r9   r@   )r¨   rW   Ú_r@   r@   rA   Ú	<setcomp>­  s      z2EfficientFormer.no_weight_decay.<locals>.<setcomp>)Znamed_parametersrr   r@   r@   rA   Úno_weight_decay«  s    zEfficientFormer.no_weight_decayFc                 C   s   t dddgd�}|S )Nz^stem)z^stages\.(\d+)N)z^norm)iŸ† )r¬   r    )Údict)r:   ZcoarseZmatcherr@   r@   rA   Úgroup_matcher¯  s
    þzEfficientFormer.group_matcherTc                 C   s   | j D ]
}||_qd S rB   )r¯   rš   )r:   ÚenableÚsr@   r@   rA   Úset_grad_checkpointing·  s    
z&EfficientFormer.set_grad_checkpointingc                 C   s   | j | jfS rB   ©r²   r³   rr   r@   r@   rA   Úget_classifier¼  s    zEfficientFormer.get_classifierc                 C   sX   || _ |d k	r|| _|dkr*t | j|¡nt ¡ | _|dkrJt | j|¡nt ¡ | _d S )Nr   )rª   r«   r0   r1   r°   rz   r²   r³   )r:   rª   r«   r@   r@   rA   Úreset_classifierÀ  s
     z EfficientFormer.reset_classifierc                 C   s
   || _ d S rB   )r´   )r:   rÅ   r@   r@   rA   Úset_distilled_trainingÇ  s    z&EfficientFormer.set_distilled_trainingc                 C   s"   |   |¡}|  |¡}|  |¡}|S rB   )r¬   r¯   rn   ro   r@   r@   rA   Úforward_featuresË  s    


z EfficientFormer.forward_features)Ú
pre_logitsc                 C   sl   | j dkr|jdd�}|  |¡}|r(|S |  |¡|  |¡ }}| jr\| jr\tj 	¡ s\||fS || d S d S )Nr¦   r   rM   r   )
r«   Úmeanr±   r²   r³   r´   rI   r4   rG   r¤   )r:   rR   rÍ   Zx_distr@   r@   rA   Úforward_headÑ  s    

zEfficientFormer.forward_headc                 C   s   |   |¡}|  |¡}|S rB   )rÌ   rÏ   ro   r@   r@   rA   rY   ß  s    

zEfficientFormer.forward)F)T)N)T)F)rZ   r[   r\   r0   r�   rk   r“   r(   r¶   r4   rG   ÚignorerÂ   rÄ   rÇ   rÉ   rÊ   rË   rÌ   ÚboolrÏ   rY   r^   r@   r@   r>   rA   r   c  s>   ï@


c                 C   sä   d| kr| S i }ddl }d}|  ¡ D ]º\}}| d¡rf| dd¡}| dd¡}| d	d
¡}| dd¡}| d|¡rz|d7 }| dd|› d�|¡}| dd|› d�|¡}| dd|› d�|¡}| dd|¡}| dd¡}|||< q$|S )z$ Remap original checkpoints -> timm zstem.0.weightr   NZpatch_embedzpatch_embed.0ú
stem.conv1zpatch_embed.1z
stem.norm1zpatch_embed.3z
stem.conv2zpatch_embed.4z
stem.norm2znetwork\.(\d+)\.proj\.weightr   znetwork.(\d+).(\d+)zstages.z
.blocks.\2znetwork.(\d+).projz.downsample.convznetwork.(\d+).normz.downsample.normzlayer_scale_([0-9])z
ls\1.gammaZ	dist_headr³   )ÚreÚitemsÚ
startswithÚreplaceÚmatchÚsub)Z
state_dictÚmodelZout_dictrÓ   Z	stage_idxrW   rX   r@   r@   rA   Ú_checkpoint_filter_fnå  s(    

rÚ   Ú c                 K   s    | ddd dddt tdddœ|–S )	Nr¥   )r   r   r   Tgffffffî?ZbicubicrÒ   rÈ   )Úurlrª   Z
input_sizerv   Zfixed_input_sizeZcrop_pctÚinterpolationrÎ   r»   Z
first_convÚ
classifierr   )rÜ   r¹   r@   r@   rA   Ú_cfg   s          ûúrß   ztimm/)Z	hf_hub_id)z!efficientformer_l1.snap_dist_in1kz!efficientformer_l3.snap_dist_in1kz!efficientformer_l7.snap_dist_in1kFc                 K   s0   |  dd ¡rtdƒ‚tt| |fdti|—Ž}|S )NZfeatures_onlyz9features_only not implemented for EfficientFormer models.Zpretrained_filter_fn)ÚgetÚRuntimeErrorr   r   rÚ   )ÚvariantÚ
pretrainedr¹   rÙ   r@   r@   rA   Ú_create_efficientformer  s      ÿþýrä   )rF   c                 K   s0   t td td dd�}tdd| it |f|Ž—ŽS )Nr   r   ©r·   r¸   r¢   Úefficientformer_l1rã   )ræ   ©rÃ   ÚEfficientFormer_depthÚEfficientFormer_widthrä   ©rã   r¹   Z
model_argsr@   r@   rA   ræ   #  s    ýræ   c                 K   s0   t td td dd�}tdd| it |f|Ž—ŽS )Nr   r   rå   Úefficientformer_l3rã   )rë   rç   rê   r@   r@   rA   rë   -  s    ýrë   c                 K   s0   t td td dd�}tdd| it |f|Ž—ŽS )Nr   r!   rå   Úefficientformer_l7rã   )rì   rç   rê   r@   r@   rA   rì   7  s    ýrì   )rÛ   )F)F)F)F).rp   Útypingr   r4   Ztorch.nnr0   Z	timm.datar   r   Ztimm.layersr   r   r   r	   r
   Z_builderr   Z_manipulater   Ú	_registryr   r   Ú__all__ré   rè   ÚModuler"   rŸ   r_   rl   rq   rt   rw   r‚   r‰   r”   r–   r™   r   rÚ   rß   Zdefault_cfgsrä   ræ   rë   rì   r@   r@   r@   rA   Ú<module>   sd   ýý=
$
!"G 
ÿÿÿù
		