U
    ¯Ê¹j!Œ  ã                   @   s^  d Z ddlZddlmZ ddlmZ ddlmZmZ ddl	m
Z
mZmZmZ ddlZddlm  mZ ddlmZ ddlmZmZ dd	lmZmZmZmZmZ d
dlmZ d
dlmZ d
dl m!Z!m"Z"m#Z# ddgZ$eG dd„ dƒƒZ%dd„ Z&G dd„ dej'ƒZ(ed\ee) e*eej+eej+ f dœdd„ƒZ,ed]e)eej+ eej+ee) f dœdd„ƒZ-eej+ej+e*ee) ee) ej+ej+dœdd„ƒZ.G dd „ d ej'ƒZ/G d!d"„ d"ej'ƒZ0G d#d$„ d$ej'ƒZ1G d%d&„ d&ej'ƒZ2G d'd„ dej'ƒZ3d(d)„ Z4e5e%d*d+�e%d,d+�e%d-d+�e%d.d/d0d1d2�e%d,dd3�e%d-dd3�e%d.d/d0ddd4�e%d5d6d7ddd4�d8�Z6d^d9d:„Z7d_d<d=„Z8e#e8d>d?d@�e8dAd?d@�e8dBd?d@�e8dCd?d@�e8d;dD�e8dEd?dFdG�e8dHd?dFdG�e8dId?dFdG�dJœƒZ9e!d`e3dKœdLdM„ƒZ:e!dae3dKœdNdO„ƒZ;e!dbe3dKœdPdQ„ƒZ<e!dce3dKœdRdS„ƒZ=e!dde3dKœdTdU„ƒZ>e!dee3dKœdVdW„ƒZ?e!dfe3dKœdXdY„ƒZ@e!dge3dKœdZd[„ƒZAdS )ha   Multi-Scale Vision Transformer v2

@inproceedings{li2021improved,
  title={MViTv2: Improved multiscale vision transformers for classification and detection},
  author={Li, Yanghao and Wu, Chao-Yuan and Fan, Haoqi and Mangalam, Karttikeya and Xiong, Bo and Malik, Jitendra and Feichtenhofer, Christoph},
  booktitle={CVPR},
  year={2022}
}

Code adapted from original Apache 2.0 licensed impl at https://github.com/facebookresearch/mvit
Original copyright below.

Modifications and timm support by / Copyright 2022, Ross Wightman
é    N)ÚOrderedDict)Ú	dataclass)ÚpartialÚreduce)ÚUnionÚListÚTupleÚOptional)Únn©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚMlpÚDropPathÚtrunc_normal_tf_Úget_norm_layerÚ	to_2tupleé   )Úbuild_model_with_cfg)Úregister_notrace_function)Úregister_modelÚregister_model_deprecationsÚgenerate_default_cfgsÚMultiScaleVitÚMultiScaleVitCfgc                   @   s¾  e Zd ZU dZeedf ed< dZeeeedf f ed< dZ	eeeedf f ed< dZ
eed	< d
Zeed< dZeed< dZeed< d
Zeed< d
Zeed< dZeed< dZeed< dZeeef ed< dZeeeeef   ed< dZeeeeef   ed< dZeeeef  ed< dZeeef ed< dZeeef ed< dZeeef ed< d Zeed!< d"Zeed#< d$Zeeeeef f ed%< d&Z eeeeef f ed'< d(Z!eed)< d*d+„ Z"dS ),r   ©é   é   é   r   .Údepthsé`   Ú	embed_dimr   Ú	num_headsç      @Ú	mlp_ratioFÚ
pool_firstTÚexpand_attnÚqkv_biasÚuse_cls_tokenÚuse_abs_posÚresidual_poolingÚconvÚmode©r   r   Ú
kernel_qkv)©r   r   ©r   r   r0   r0   Ústride_qNÚ	stride_kv©é   r4   Ústride_kv_adaptive©é   r7   Úpatch_kernelÚpatch_strideÚpatch_paddingÚmaxÚ	pool_typeÚspatialÚrel_pos_typeZgeluÚ	act_layerZ	layernormÚ
norm_layerg�íµ ÷Æ°>Únorm_epsc                    sþ   t ˆjƒ}tˆjttfƒs6t‡fdd„t|ƒD ƒƒˆ_t ˆjƒ|ksHt‚tˆjttfƒstt‡fdd„t|ƒD ƒƒˆ_t ˆjƒ|ks†t‚ˆj	d k	rúˆj
d krúˆj	‰ g }t|ƒD ]B‰tˆjˆ ƒdkrà‡ ‡‡fdd„tt ˆ ƒƒD ƒ‰ | tˆ ƒ¡ q¬t|ƒˆ_
d S )Nc                 3   s   | ]}ˆ j d |  V  qdS ©r   N)r!   ©Ú.0Úi©Úself© úQ/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/mvitv2.pyÚ	<genexpr>@   s     z1MultiScaleVitCfg.__post_init__.<locals>.<genexpr>c                 3   s   | ]}ˆ j d |  V  qdS rB   )r"   rC   rF   rH   rI   rJ   D   s     r   c                    s(   g | ] }t ˆ | ˆjˆ |  d ƒ‘qS ©r   )r;   r1   )rD   Úd©Z
_stride_kvrE   rG   rH   rI   Ú
<listcomp>L   s   ÿz2MultiScaleVitCfg.__post_init__.<locals>.<listcomp>)Úlenr   Ú
isinstancer!   ÚtupleÚlistÚrangeÚAssertionErrorr"   r5   r2   Úminr1   Úappend)rG   Ú
num_stagesZpool_kv_striderH   rM   rI   Ú__post_init__=   s"    

þzMultiScaleVitCfg.__post_init__)#Ú__name__Ú
__module__Ú__qualname__r   r   ÚintÚ__annotations__r!   r   r"   r$   Úfloatr%   Úboolr&   r'   r(   r)   r*   r,   Ústrr.   r1   r	   r2   r5   r8   r9   r:   r<   r>   r?   r@   rA   rX   rH   rH   rH   rI   r   #   s0   
c                 C   s   t tj| dƒS )Nr   )r   ÚoperatorÚmul)ÚiterablerH   rH   rI   ÚprodT   s    rd   c                       s>   e Zd ZdZd‡ fdd„	Zeejee	 f d	œd
d„Z
‡  ZS )Ú
PatchEmbedz
    PatchEmbed.
    r   é   r6   r3   r-   c                    s$   t ƒ  ¡  tj|||||d�| _d S )N)Zkernel_sizeÚstrideÚpadding)ÚsuperÚ__init__r
   ÚConv2dÚproj)rG   Údim_inÚdim_outÚkernelrg   rh   ©Ú	__class__rH   rI   rj   ]   s    
ûzPatchEmbed.__init__©Úreturnc                 C   s*   |   |¡}| d¡ dd¡|jdd … fS )Nr   r   éþÿÿÿ)rl   ÚflattenÚ	transposeÚshape©rG   ÚxrH   rH   rI   Úforwardo   s    
zPatchEmbed.forward)r   rf   r6   r3   r-   )rY   rZ   r[   Ú__doc__rj   r   ÚtorchÚTensorr   r\   rz   Ú__classcell__rH   rH   rp   rI   re   X   s        úre   T)Ú	feat_sizeÚhas_cls_tokenrs   c                 C   s‚   |\}}|rP| d d …d d …d d…d d …f | d d …d d …dd …d d …f  }} nd }|   d||| jd ¡ dddd¡ ¡ } | |fS )Nr   éÿÿÿÿr   r   r   )Úreshaperw   ÚpermuteÚ
contiguous)ry   r   r€   ÚHÚWÚcls_tokrH   rH   rI   Úreshape_pre_poolu   s    D&rˆ   )r"   r‡   rs   c                 C   sh   | j d | j d g}| j d | j d  }|  d|| j d |¡ dd¡} |d k	r`tj|| fdd�} | |fS )Nr   r   r�   r   ©Údim)rw   r‚   rv   r|   Úcat)ry   r"   r‡   r   ZL_pooledrH   rH   rI   Úreshape_post_pool„   s    rŒ   )ÚattnÚqr€   Úq_sizeÚk_sizeÚ	rel_pos_hÚ	rel_pos_wc                 C   s¾  |rdnd}|\}}	|\}
}t |
| dƒ}t ||
 dƒ}tj||jd� d¡| tj|
|jd� d¡|  }||
d | 7 }t ||	 dƒ}t |	| dƒ}tj|	|jd� d¡| tj||jd� d¡|  }||d | 7 }|| ¡  }|| ¡  }|j\}}}}|dd…dd…|d…f  ||||	|¡}t d||¡}t d||¡}| dd…dd…|d…|d…f  	|d||	|
|¡| d¡ | d	¡  	|d||	 |
| ¡| dd…dd…|d…|d…f< | S )
z1
    Spatial Relative Positional Embeddings.
    r   r   g      ð?)Údevicer�   Nzbyhwc,hkc->byhwkzbyhwc,wkc->byhwkrt   )
r;   r|   Zaranger“   Z	unsqueezeÚlongrw   r‚   ZeinsumÚview)r�   rŽ   r€   r�   r�   r‘   r’   Zsp_idxZq_hZq_wZk_hZk_wZ	q_h_ratioZ	k_h_ratioZdist_hZ	q_w_ratioZ	k_w_ratioZdist_wZrel_hZrel_wÚBZn_headÚq_NrŠ   Zr_qrH   rH   rI   Úcal_rel_pos_type’   sF    ÿÿÿÿ(.ÿþ   ýÿr˜   c                       sH   e Zd Zddddddddddejf‡ fdd„	Zee dœd	d
„Z‡  Z	S )ÚMultiScaleAttentionPoolFirsté   Tr+   r/   r=   c              	      sÜ  t ƒ  ¡  || _|| _|| | _| jd | _|| _tdd„ |D ƒƒ}tdd„ |D ƒƒ}tj	|||d�| _
tj	|||d�| _tj	|||d�| _t 	||¡| _t|ƒdkr¶t|	ƒdkr¶d }t|ƒdkrÒt|
ƒdkrÒd }|| _|dk| _d\| _| _| _d\| _| _| _|d	k�rZ|d
k�rtjntj}|�r6|||	|ƒ| _|�r|||
|ƒ| _|||
|ƒ| _n²|dk�sn|dk�rþ|dk�r€|| n|}|�r®tj||||	||dd�| _||ƒ| _|�rtj||||
||dd�| _||ƒ| _tj||||
||dd�| _||ƒ| _ntd|› �ƒ‚|| _| jdk�rÒ|d |d k�s4t‚|d }t|	ƒdk�rV||	d  n|}t|
ƒdk�rt||
d  n|}dt||ƒ d }t t  !|| j¡¡| _"t t  !|| j¡¡| _#t$| j"dd� t$| j#dd� || _%d S )Nç      à¿c                 S   s   g | ]}t |d  ƒ‘qS ©r   ©r\   ©rD   rŽ   rH   rH   rI   rN   Ý   s     z9MultiScaleAttentionPoolFirst.__init__.<locals>.<listcomp>c                 S   s   g | ]}t |d  ƒ‘qS rœ   r�   ©rD   ÚkvrH   rH   rI   rN   Þ   s     ©Úbiasr   Úconv_unshared©NNN©Úavgr;   r;   r+   F©rg   rh   Úgroupsr¢   úUnsupported model r=   r   r   ç{®Gáz”?©Ústd)&ri   rj   r"   rn   Úhead_dimÚscaler€   rQ   r
   ÚLinearrŽ   ÚkÚvrl   rd   r,   ÚunsharedÚpool_qÚpool_kÚpool_vÚnorm_qÚnorm_kÚnorm_vÚ	MaxPool2dÚ	AvgPool2drk   ÚNotImplementedErrorr>   rT   rO   r;   Ú	Parameterr|   Úzerosr‘   r’   r   r*   ©rG   rŠ   rn   r   r"   r'   r,   Úkernel_qÚ	kernel_kvr1   r2   r€   r>   r*   r@   Z	padding_qZ
padding_kvZpool_opZdim_convÚsizer�   Zkv_sizeZ
rel_sp_dimrp   rH   rI   rj   Æ   s”    



ù	
ù	
ù	z%MultiScaleAttentionPoolFirst.__init__©r   c                 C   s¨  |j \}}}| jrdn| j}| |||d¡ dddd¡}| } }}	| jd k	r~t||| jƒ\}}
|  |¡}t|| j|
ƒ\}}n|}| j	d k	r–|  	|¡}| j
d k	rÐt||| jƒ\}}|  
|¡}t|| j|ƒ\}}n|}| jd k	rè|  |¡}| jd k	�r$t|	|| jƒ\}	}|  |	¡}	t|	| j|ƒ\}	}n|}| jd k	�r>|  |	¡}	|d |d  t| jƒ }| dd¡ ||d¡}|  |¡ ||| jd¡ dd¡}|d |d  t| jƒ }| dd¡ ||d¡}|  |¡ ||| jd¡}|d |d  t| jƒ }|	 dd¡ ||d¡}	|  |	¡ ||| jd¡ dd¡}	|| j | }| jdk�rZt||| j||| j| jƒ}|jdd�}||	 }| j�r~|| }| dd¡ |d| j¡}|  |¡}||fS )Nr   r�   r   r   r   r=   r‰   )rw   r²   r"   r‚   rƒ   r³   rˆ   r€   rŒ   r¶   r´   r·   rµ   r¸   r\   rv   rŽ   r°   r±   r®   r>   r˜   r‘   r’   Úsoftmaxr*   rn   rl   )rG   ry   r   r–   ÚNÚ_Zfold_dimrŽ   r°   r±   Úq_tokr�   Úk_tokr�   Úv_tokZv_sizer—   Zk_NZv_Nr�   rH   rH   rI   rz   *  sh    









  ù	
z$MultiScaleAttentionPoolFirst.forward©
rY   rZ   r[   r
   Ú	LayerNormrj   r   r\   rz   r~   rH   rH   rp   rI   r™   Å   s   ñdr™   c                       sH   e Zd Zddddddddddejf‡ fdd„	Zee dœd	d
„Z‡  Z	S )ÚMultiScaleAttentionrš   Tr+   r/   r=   c              	      sº  t ƒ  ¡  || _|| _|| | _| jd | _|| _tdd„ |D ƒƒ}tdd„ |D ƒƒ}tj	||d |d�| _
t 	||¡| _t|ƒdkr–t|	ƒdkr–d }t|ƒdkr²t|
ƒdkr²d }|| _|dk| _d	\| _| _| _d	\| _| _| _|d
k�r8|dkrútjntj}|�r|||	|ƒ| _|�rê|||
|ƒ| _|||
|ƒ| _n²|dk�sL|dk�rÜ|dk�r^|| n|}|�rŒtj||||	||dd�| _||ƒ| _|�rêtj||||
||dd�| _||ƒ| _tj||||
||dd�| _||ƒ| _ntd|› �ƒ‚|| _| jdk�r°|d |d k�st‚|d }t|	ƒdk�r4||	d  n|}t|
ƒdk�rR||
d  n|}dt||ƒ d }t t || j¡¡| _ t t || j¡¡| _!t"| j dd� t"| j!dd� || _#d S )Nr›   c                 S   s   g | ]}t |d  ƒ‘qS rœ   r�   rž   rH   rH   rI   rN   ‡  s     z0MultiScaleAttention.__init__.<locals>.<listcomp>c                 S   s   g | ]}t |d  ƒ‘qS rœ   r�   rŸ   rH   rH   rI   rN   ˆ  s     r   r¡   r   r£   r¤   r¥   r;   r+   Fr§   r©   r=   r   r   rª   r«   )$ri   rj   r"   rn   r­   r®   r€   rQ   r
   r¯   Úqkvrl   rd   r,   r²   r¶   r·   r¸   r³   r´   rµ   r¹   rº   rk   r»   r>   rT   rO   r;   r¼   r|   r½   r‘   r’   r   r*   r¾   rp   rH   rI   rj   p  s�    



ù	
ù	
ù	zMultiScaleAttention.__init__rÂ   c                 C   sÄ  |j \}}}|  |¡ ||d| jd¡ ddddd¡}|jdd�\}}}	| jd k	r€t||| jƒ\}}
|  |¡}t	|| j|
ƒ\}}n|}| j
d k	r˜|  
|¡}| jd k	rÒt||| jƒ\}}|  |¡}t	|| j|ƒ\}}n|}| jd k	rê|  |¡}| jd k	�r$t|	|| jƒ\}	}|  |	¡}	t	|	| j|ƒ\}	}| jd k	�r:|  |	¡}	|| j | dd¡ }| jd	k�rvt||| j||| j| jƒ}|jdd�}||	 }| j�rš|| }| dd¡ |d| j¡}|  |¡}||fS )
Nr   r�   r   r   r   r4   r‰   rt   r=   )rw   rÌ   r‚   r"   rƒ   Zunbindr³   rˆ   r€   rŒ   r¶   r´   r·   rµ   r¸   r®   rv   r>   r˜   r‘   r’   rÃ   r*   rn   rl   )rG   ry   r   r–   rÄ   rÅ   rÌ   rŽ   r°   r±   rÆ   r�   rÇ   r�   rÈ   r�   rH   rH   rI   rz   Ò  sR    (









ù	
zMultiScaleAttention.forwardrÉ   rH   rH   rp   rI   rË   o  s   ñbrË   c                       s`   e Zd Zdddejddddddddddf‡ fdd	„	Zee d
œdd„Zee d
œdd„Z	‡  Z
S )ÚMultiScaleBlockr#   Tç        r/   r+   Fr=   c                    sL  t ƒ  ¡  ||k}|| _|| _|| _||ƒ| _|rB|rBt ||¡nd | _|rŠt	|ƒdkrŠdd„ |D ƒ}|}dd„ |D ƒ}t 
|||¡| _nd | _|r˜|n|}|r¤tnt}|||||||	|
|||||||d�| _|dkrÞt|ƒnt ¡ | _||ƒ| _|}|�r|�st ||¡nd | _t|t|| ƒ|d�| _|dk�r>t|ƒnt ¡ | _d S )Nr   c                 S   s    g | ]}|d kr|d  n|‘qS rK   rH   )rD   ÚsrH   rH   rI   rN   (  s     z,MultiScaleBlock.__init__.<locals>.<listcomp>c                 S   s   g | ]}t |d  ƒ‘qS rœ   r�   )rD   ÚskiprH   rH   rI   rN   *  s     )r"   r   r'   r¿   rÀ   r1   r2   r@   r€   r,   r>   r*   rÎ   )Zin_featuresZhidden_featuresZout_features)ri   rj   rŠ   rn   r€   Únorm1r
   r¯   Úshortcut_proj_attnrd   r¹   Úshortcut_pool_attnr™   rË   r�   r   ÚIdentityÚ
drop_path1Únorm2Úshortcut_proj_mlpr   r\   ÚmlpÚ
drop_path2)rG   rŠ   rn   r"   r   r$   r'   Ú	drop_pathr@   r¿   rÀ   r1   r2   r,   r€   r&   r%   r>   r*   Zproj_neededZkernel_skipZstride_skipZpadding_skipZatt_dimZ
attn_layerZmlp_dim_outrp   rH   rI   rj   	  sR    

ò

ýzMultiScaleBlock.__init__rÂ   c           	      C   sÂ   | j d kr|S | jrL|d d …d d…d d …f |d d …dd …d d …f  }}nd }|j\}}}|\}}| ||||¡ dddd¡ ¡ }|   |¡}| ||d¡ dd¡}|d k	r¾tj||fdd�}|S )Nr   r   r   r   r�   r‰   )	rÓ   r€   rw   r‚   rƒ   r„   rv   r|   r‹   )	rG   ry   r   r‡   r–   ÚLÚCr…   r†   rH   rH   rI   Ú_shortcut_poolM  s    
8 
zMultiScaleBlock._shortcut_poolc                 C   sŠ   |   |¡}| jd kr|n|  |¡}|  ||¡}|  ||¡\}}||  |¡ }|  |¡}| jd krd|n|  |¡}||  |  |¡¡ }||fS ©N)	rÑ   rÒ   rÝ   r�   rÕ   rÖ   r×   rÙ   rØ   )rG   ry   r   Zx_normZ
x_shortcutZfeat_size_newrH   rH   rI   rz   ]  s    

zMultiScaleBlock.forward)rY   rZ   r[   r
   rÊ   rj   r   r\   rÝ   rz   r~   rH   rH   rp   rI   rÍ     s"   íDrÍ   c                       sN   e Zd Zddddddddddddejdf‡ fdd	„	Zee d
œdd„Z‡  Z	S )ÚMultiScaleVitStager#   Tr+   r/   Fr=   rÎ   c                    sÜ   t ƒ  ¡  d| _t ¡ | _|r*|f| }n|f|d  |f }t|ƒD ]Š}t||| |||||	|
|dkrl|nd||||||||t|t	t
fƒr”|| n|d�}|| }| j |¡ |dkrFt
dd„ t||ƒD ƒƒ}qF|| _d S )NFr   r   r/   )rŠ   rn   r"   r   r$   r'   r¿   rÀ   r1   r2   r,   r€   r%   r>   r*   r&   r@   rÚ   c                 S   s   g | ]\}}|| ‘qS rH   rH   )rD   rÁ   rg   rH   rH   rI   rN   ¤  s     z/MultiScaleVitStage.__init__.<locals>.<listcomp>)ri   rj   Úgrad_checkpointingr
   Ú
ModuleListÚblocksrS   rÍ   rP   rR   rQ   rV   Úzipr   )rG   rŠ   rn   Údepthr"   r   r$   r'   r,   r¿   rÀ   r1   r2   r€   r&   r%   r>   r*   r@   rÚ   Zout_dimsrE   Zattention_blockrp   rH   rI   rj   m  s@    

îzMultiScaleVitStage.__init__rÂ   c                 C   sF   | j D ]6}| jr.tj ¡ s.t |||¡\}}q|||ƒ\}}q||fS rÞ   )râ   rà   r|   ÚjitZis_scriptingÚ
checkpoint)rG   ry   r   ZblkrH   rH   rI   rz   ¨  s
    
zMultiScaleVitStage.forwardrÉ   rH   rH   rp   rI   rß   k  s    	ì;rß   c                	       s¶   e Zd ZdZdeeeef eee ee	e	dœ‡ fdd	„Z
d
d„ Zejjdd„ ƒZejjd dd„ƒZejjd!dd„ƒZejjdd„ ƒZd"dd„Zdd„ Zd#edœdd„Zdd„ Z‡  ZS )$r   a¸  
    Improved Multiscale Vision Transformers for Classification and Detection
    Yanghao Li*, Chao-Yuan Wu*, Haoqi Fan, Karttikeya Mangalam, Bo Xiong, Jitendra Malik,
        Christoph Feichtenhofer*
    https://arxiv.org/abs/2112.01526

    Multiscale Vision Transformers
    Haoqi Fan*, Bo Xiong*, Karttikeya Mangalam*, Yanghao Li*, Zhicheng Yan, Jitendra Malik,
        Christoph Feichtenhofer*
    https://arxiv.org/abs/2104.11227
    ©éà   rè   r   Néè  rÎ   )ÚcfgÚimg_sizeÚin_chansÚglobal_poolÚnum_classesÚdrop_path_rateÚ	drop_ratec                    s–  t ƒ  ¡  t|ƒ}tt|jƒ|jd�}|| _|| _|d krH|j	rDdnd}|| _
t|jƒ| _|j| _|jd }	t||	|j|j|jd�| _|d |jd  |d |jd  f}
t|
ƒ}|j	rÞt t dd|	¡¡| _d| _|d }nd| _d | _|}|j�rt t d||	¡¡| _nd | _t|jƒ}|
}dd„ t d|t|jƒ¡  |j¡D ƒ}t !¡ | _"t#|ƒD ]¦}|j�rr|j| }n|jt$|d |d ƒ }t%|	||j| |j&| ||j'|j(|j)|j*|j|j+|j+|j,| |j-| |j	|j.|j/||| d	�}|}	|j0}| j" 1|¡ �qZ|	| _2||	ƒ| _3t 4t5d
t 6| j¡fd|dk�r@t 7| j2|¡nt 8¡ fgƒ¡| _9| jd k	�rlt:| jdd� | jd k	�r†t:| jdd� |  ;| j<¡ d S )N)ÚepsÚtokenr¦   r   )rm   rn   ro   rg   rh   r   c                 S   s   g | ]}|  ¡ ‘qS rH   )Útolist)rD   ry   rH   rH   rI   rN   î  s     z*MultiScaleVit.__init__.<locals>.<listcomp>)rŠ   rn   rä   r"   r   r$   r'   r,   r%   r&   r¿   rÀ   r1   r2   r€   r>   r*   r@   rÚ   ÚdropÚfcrª   r«   )=ri   rj   r   r   r   r@   rA   rî   rð   r(   rí   rQ   r   r&   r!   re   r8   r9   r:   Úpatch_embedrd   r
   r¼   r|   r½   Ú	cls_tokenÚnum_prefix_tokensr)   Ú	pos_embedrO   ZlinspaceÚsumÚsplitrá   ÚstagesrS   rU   rß   r"   r$   r'   r,   r%   r.   r1   r2   r>   r*   r   rV   Únum_featuresÚnormÚ
Sequentialr   ÚDropoutr¯   rÔ   Úheadr   ÚapplyÚ_init_weights)rG   rê   rë   rì   rí   rî   rï   rð   r@   r!   Z
patch_dimsZnum_patchesZpos_embed_dimrW   r   ZdprrE   rn   Ústagerp   rH   rI   rj   ¾  s’    


û$

&
í
"þ
zMultiScaleVit.__init__c                 C   sD   t |tjƒr@t|jdd� t |tjƒr@|jd k	r@tj |jd¡ d S )Nrª   r«   rÎ   )rP   r
   r¯   r   Úweightr¢   ÚinitZ	constant_)rG   ÚmrH   rH   rI   r    s    zMultiScaleVit._init_weightsc                 C   s   dd„ |   ¡ D ƒS )Nc                    s*   h | ]"\‰ }t ‡ fd d„dD ƒƒrˆ ’qS )c                 3   s   | ]}|ˆ kV  qd S rÞ   rH   )rD   Ún©r°   rH   rI   rJ   $  s     z:MultiScaleVit.no_weight_decay.<locals>.<setcomp>.<genexpr>)rù   r‘   r’   r÷   )Úany)rD   rÅ   rH   r	  rI   Ú	<setcomp>#  s    ÿz0MultiScaleVit.no_weight_decay.<locals>.<setcomp>)Znamed_parametersrF   rH   rH   rI   Úno_weight_decay!  s    zMultiScaleVit.no_weight_decayFc                 C   s   t dddgd�}|S )Nz^patch_embed)z^stages\.(\d+)N)z^norm)iŸ† )Ústemrâ   )Údict)rG   ZcoarseZmatcherrH   rH   rI   Úgroup_matcher&  s
    þzMultiScaleVit.group_matcherTc                 C   s   | j D ]
}||_qd S rÞ   )rü   rà   )rG   ÚenablerÏ   rH   rH   rI   Úset_grad_checkpointing.  s    
z$MultiScaleVit.set_grad_checkpointingc                 C   s   | j jS rÞ   )r  rõ   rF   rH   rH   rI   Úget_classifier3  s    zMultiScaleVit.get_classifierc              	   C   sV   || _ |d k	r|| _t tdt | j¡fd|dkr@t | j|¡nt 	¡ fgƒ¡| _
d S )Nrô   rõ   r   )rî   rí   r
   rÿ   r   r   rð   r¯   rý   rÔ   r  )rG   rî   rí   rH   rH   rI   Úreset_classifier7  s     þzMultiScaleVit.reset_classifierc                 C   s‚   |   |¡\}}|j\}}}| jd k	rF| j |dd¡}tj||fdd�}| jd k	rZ|| j }| jD ]}|||ƒ\}}q`|  |¡}|S )Nr�   r   r‰   )	rö   rw   r÷   Úexpandr|   r‹   rù   rü   rþ   )rG   ry   r   r–   rÄ   rÜ   Z
cls_tokensr  rH   rH   rI   Úforward_features@  s    




zMultiScaleVit.forward_features)Ú
pre_logitsc                 C   sP   | j r>| j dkr.|d d …| jd …f  d¡}n|d d …df }|rF|S |  |¡S )Nr¦   r   r   )rí   rø   Úmeanr  )rG   ry   r  rH   rH   rI   Úforward_headQ  s
    
zMultiScaleVit.forward_headc                 C   s   |   |¡}|  |¡}|S rÞ   )r  r  rx   rH   rH   rI   rz   Y  s    

zMultiScaleVit.forward)rç   r   Nré   rÎ   rÎ   )F)T)N)F)rY   rZ   r[   r{   r   r   r\   r	   r`   r^   rj   r  r|   rå   Úignorer  r  r  r  r  r  r_   r  rz   r~   rH   rH   rp   rI   r   ±  s:         ø
ø]


	c           	         s   d| kr| S dd l }d| kr$| d } t|dd ƒ}t|ddƒ}|d k	sLtdƒ‚i ‰d‰ t|ƒD ]4\‰}ˆ ‡ ‡fdd	„tˆ ˆ | ƒD ƒ¡ ˆ |7 ‰ q\i }|  ¡ D ]\\}}| d
‡fdd„|¡}|rÐ| dd|¡}n| dd|¡}d|krò| dd¡}|||< qž|S )Nzstages.0.blocks.0.norm1.weightr   Zmodel_stater   r&   Tz3model requires depth attribute to remap checkpointsc                    s   i | ]}|ˆ|ˆ  f“qS rH   rH   rC   )Ú	block_idxÚ	stage_idxrH   rI   Ú
<dictcomp>m  s      z(checkpoint_filter_fn.<locals>.<dictcomp>zblocks\.(\d+)c                    s4   dˆ t |  d¡ƒ d › dˆ t |  d¡ƒ d › �S )Nzstages.r   r   z.blocks.)r\   Úgroup)ry   )Ú	depth_maprH   rI   Ú<lambda>t  ó    z&checkpoint_filter_fn.<locals>.<lambda>z stages\.(\d+).blocks\.(\d+).projz&stages.\1.blocks.\2.shortcut_proj_attnz%stages.\1.blocks.\2.shortcut_proj_mlpr  zhead.projectionúhead.fc)	ÚreÚgetattrrT   Ú	enumerateÚupdaterS   ÚitemsÚsubÚreplace)	Z
state_dictÚmodelr"  r   r&   rL   Zout_dictr°   r±   rH   )r  r  r  rI   Úcheckpoint_filter_fn_  s6    $

ý
r*  )r   r   é   r   )r   )r   r   é   r   r   )r   é   é$   r4   é�   r   F)r   r!   r"   r&   )r   r(   )r   r!   r"   r(   r&   )r4   rš   é<   rš   éÀ   r   )Úmvitv2_tinyÚmvitv2_smallÚmvitv2_baseÚmvitv2_largeÚmvitv2_small_clsÚmvitv2_base_clsÚmvitv2_large_clsÚmvitv2_huge_clsc                 K   sF   |  dd ¡rtdƒ‚tt| |f|s*t|  nt| ttdd�dœ|—ŽS )NZfeatures_onlyzGfeatures_only not implemented for Multiscale Vision Transformer models.T)Zflatten_sequential)Z	model_cfgZpretrained_filter_fnZfeature_cfg)ÚgetÚRuntimeErrorr   r   Ú
model_cfgsr*  r  )ÚvariantZcfg_variantÚ
pretrainedÚkwargsrH   rH   rI   Ú_create_mvitv2¶  s    ýúùr@  Ú c                 K   s    | ddd ddt tddddœ|–S )	Nré   )r   rè   rè   gÍÌÌÌÌÌì?Zbicubiczpatch_embed.projr!  T)Úurlrî   Z
input_sizeZ	pool_sizeZcrop_pctÚinterpolationr  r¬   Z
first_convÚ
classifierZfixed_input_sizer   )rB  r?  rH   rH   rI   Ú_cfgÅ  s         úùrE  zDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_T_in1k.pythztimm/)rB  Ú	hf_hub_idzDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_S_in1k.pythzDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_B_in1k.pythzDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_L_in1k.pyth)rB  zEhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_B_in21k.pythiàJ  )rB  rF  rî   zEhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_L_in21k.pythzEhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_H_in21k.pyth)zmvitv2_tiny.fb_in1kzmvitv2_small.fb_in1kzmvitv2_base.fb_in1kzmvitv2_large.fb_in1kr6  zmvitv2_base_cls.fb_inw21kzmvitv2_large_cls.fb_inw21kzmvitv2_huge_cls.fb_inw21krr   c                 K   s   t dd| i|—ŽS )Nr2  r>  )r2  ©r@  ©r>  r?  rH   rH   rI   r2  ì  s    r2  c                 K   s   t dd| i|—ŽS )Nr3  r>  )r3  rG  rH  rH   rH   rI   r3  ñ  s    r3  c                 K   s   t dd| i|—ŽS )Nr4  r>  )r4  rG  rH  rH   rH   rI   r4  ö  s    r4  c                 K   s   t dd| i|—ŽS )Nr5  r>  )r5  rG  rH  rH   rH   rI   r5  û  s    r5  c                 K   s   t dd| i|—ŽS )Nr6  r>  )r6  rG  rH  rH   rH   rI   r6     s    r6  c                 K   s   t dd| i|—ŽS )Nr7  r>  )r7  rG  rH  rH   rH   rI   r7    s    r7  c                 K   s   t dd| i|—ŽS )Nr8  r>  )r8  rG  rH  rH   rH   rI   r8  
  s    r8  c                 K   s   t dd| i|—ŽS )Nr9  r>  )r9  rG  rH  rH   rH   rI   r9    s    r9  )T)N)NF)rA  )F)F)F)F)F)F)F)F)Br{   ra   Úcollectionsr   Údataclassesr   Ú	functoolsr   r   Útypingr   r   r   r	   r|   Ztorch.utils.checkpointÚutilsræ   r
   Z	timm.datar   r   Ztimm.layersr   r   r   r   r   Z_builderr   Z_features_fxr   Ú	_registryr   r   r   Ú__all__r   rd   ÚModulere   r\   r_   r}   rˆ   rŒ   r˜   r™   rË   rÍ   rß   r   r*  r  r<  r@  rE  Zdefault_cfgsr2  r3  r4  r5  r6  r7  r8  r9  rH   rH   rH   rI   Ú<module>   s  0 ýü ýüù2 + cF /-ÿÿÿüþþûûà*

þÿÿÿýýýì