U
    ¯Ê¹jˆÛ ã                ¤   @   sL  d Z ddlZddlZddlmZ ddlmZ ddlmZm	Z	m
Z
mZmZmZmZmZmZmZ zddlmZ W n  ek
rŒ   ddlmZ Y nX ddlZddlmZ ddlm  mZ ddlZddlmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$ dd	l%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 d
dl5m6Z6 d
dl7m8Z8m9Z9m:Z: d
dl;m<Z<m=Z=m>Z> dgZ?e @eA¡ZBG dd„ dejCƒZDG dd„ dejCƒZEG dd„ dejCƒZFG dd„ dejCƒZGG dd„ dejCƒZHG dd„ dejCƒZIG dd„ dejCƒZJ�d¹ejCeKddœdd„ZL�dºejCeKeMdd!œd"d#„ZN�d»ejCeKddœd$d%„ZO�d¼eKeMe	d'œd(d)„ZP�d½ejQejQeReeReRf eKeSejQd-œd.d/„ZTe U¡ �d¾eJeKeKdd0œd1d2„ƒZV�d¿e
eKejQf eJeKe
eKejQf d4œd5d6„ZWe
eKejQf eJe
eKejQf d7œd8d9„ZX�dÀe
eKejQf eJeSeKeSe
eKejQf d;œd<d=„ZY�dÁeKe
eKef d>œd?d@„ZZeZdAdB�eZƒ eZdAdB�eZdCdAd:dD�eZdEdAd:dFdGdH�eZdIdAd:dD�eZdJdAd:dFdGdH�eZdKdAd:dD�eZdLdAd:dFdGdH�eZdMdAd:dD�eZdNdAd:dFdGdH�eZdOdAd:dD�eZdPdAd:dFdGdH�eZdQdAd:dD�eZdRdAd:dD�eZdSdAd:dFdGdH�eZdTdAdU�eZdVdAdFdGdW�eZdXdAdFdGdW�eZdYdAd:dD�eZdZdAd:dFdGdH�eZd[dAd:dD�eZd\dAd:dFdGdH�eZd]dAd:dD�eZd^dAd:dFdGdH�eZdd_�eZdd_�eZdd_�eZdd_�eZdAdd`�eZdAdd`�eZdAdd`�eZdAdd`�eZdAdd`�eZdadAd:dbdc�eZdddAd:dbdc�eZdedAd:dbdc�eZdfdAd:dbdc�eZdgdAd:dbdc�eZdhdAd:dbdc�eZdidAd:dbdc�eZdjd:dAdk�eZdld:dAdk�eZdmdAee ddn�eZdodAee ddn�eZdpdAee ddn�eZdqdAee ddn�eZdrdAdsee ddtdGdu�eZdvdAdsee ddtdGdu�eZdwdAdsee ddtdGdu�eZdxdAdsee ddtdGdu�eZdydAdsee ddtdGdu�eZdzdAdsee ddtdGdu�eZd{dAdsee ddtdGdu�eZd|dAdsee ddtdGdu�eZd}dAd~dd€d�d‚dƒ�eZd„dAd~dd€d�d…�eZd†dAdU�eZdAd‡dˆd‰dŠ�eZdAd‹dˆdŒ�eZdAdFdˆd�dŽ�eZƒ eZdAe#e$d��eZdAe#e$dGdFd��eZdAe#e$dGd‘d��eZdAe#e$dˆd’�eZdAe#e$dGdFd�d“�eZdAe!e"dGd’�eZdAe!e"dGd”d�d“�eZdAe#e$dGd’�eZdAe#e$dGd”d�d“�eZe#e$d•�eZdAe#e$dˆdFd�d“�eZdAe#e$dˆd’�eZdAe#e$dˆdFd�d“�eZdAe#e$dGd’�eZdAe#e$dGd”d�d“�eZdAe#e$d��eZdAe#e$dGd’�eZdAe#e$dGdFd�d“�eZdAe!e"dGd’�eZdAe!e"dGd”d�d“�eZdAe#e$dGd’�eZde#e$dGd”d�d“�eZdAe#e$d��eZdAe#e$d��eZdAe#e$dGdFd�d“�eZdAe#e$dGd’�eZe#e$d‰d–�eZdAe#e$d‰d—�eZdAe!e"dGd‰d˜�eZdAe#e$dGd‰d˜�eZe#e$d‰d–�eZdAe#e$d‰d—�eZdAe#e$dGd‰d˜�eZd™dše#e$d›dœ�eZd�dše#e$dGd›dž�eZdŸdše!e"dGd dž�eZd¡dše#e$dGd¢dž�eZd£dše#e$dGd¢dž�eZd¤dše#e$dGd¥dž�eZd¦dše#e$dGd›dž�eZd§dše#e$dGd‹d›d¨�eZd©dše#e$dGd›dž�eZdªdše#e$dGd dž�eZd«dše#e$dGd›dž�eZd¬dšd­e#e$dGd d®�eZd¯dšd­e#e$dGd¢d®�eZd°dše#e$d­dGd±d¢d²�eZd³d´dµd­e#e$dGd›d¶�eZd·d¸dµd­e#e$dGd›d¶�eZd¹dºdµd­e#e$dGd d¶�eZd»d¼dµd­e#e$dGd¢d¶�eZd½d­e#e$d›d¾�eZd¿d­e#e$d›d¾�eZdÀd­e#e$dGd dÁ�eZdÂdšd­e#e$dGd”d dÃ�eZdd‹dˆdÄ�eZdd‡dˆdÄ�eZdd_�eZdd_�eZdd_�eZdAdÅe#e$dÆdGdÇ�eZdAdÅe#e$d”dGd�dÈ�eZdAdÅe#e$dÆdGdÇ�eZdAdÅe#e$d”dGd�dÈ�eZdÉd:dAd‡dˆdÊ�eZdËd:dAd‡dˆdÊ�eZdÌd:dAd‡dˆdÊ�eZdÍd:dAd‡dˆdÊ�eZdÎd:dAd‡dˆdÊ�eZdÏd:dAd‡dˆdÊ�eZdÐd:dAd‡dˆdbdÑ�eZdÒd:dAd‡dˆdbdÑ�eZdÓd:dAd‡dˆdÊ�eZdÔd:dAd‡dˆdÊ�eZdÕd:dAd‡dˆdÊ�eZdÖd:dAd‡dˆdbdÑ�eZd×d:dAd‡dˆdbdÑ�eZdd_�eZdd_�eZdd_�eZdØdAdµee ddÙ�eZdÚdAdµee ddÙ�eZdÛdAdµee ddÙ�eZdÜdµee ddÝ�eZdÞdµee ddÝ�eZdßdµd‘dGee ddà�eZdádµee ddÝ�eZdâdšddã�eZdädšd‹ddå�eZdædšdFddå�eZdçdšdèddå�eZdédšd‹ddå�eZdêdšdFddå�eZdëdšddã�eZdìdšdFddå�eZd‹dí�eZd‹dí�eZd‹dí�eZd‹dí�eZd‹dí�dîœ¢Z[dïdðdñdòdódôdõdöd÷dødùgZ\e[ ]dúdû„ e\D ƒ¡ e<e[ƒZ[�dÂeKeSeJdüœdýdþ„Z^e=�dÃeSeJdÿœ�d �d„ƒZ_e=�dÄeSeJdÿœ�d�d„ƒZ`e=�dÅeSeJdÿœ�d�d„ƒZae=�dÆeSeJdÿœ�d�d„ƒZbe=�dÇeSeJdÿœ�d�d	„ƒZce=�dÈeSeJdÿœ�d
�d„ƒZde=�dÉeSeJdÿœ�d�d„ƒZee=�dÊeSeJdÿœ�d�d„ƒZfe=�dËeSeJdÿœ�d�d„ƒZge=�dÌeSeJdÿœ�d�d„ƒZhe=�dÍeSeJdÿœ�d�d„ƒZie=�dÎeSeJdÿœ�d�d„ƒZje=�dÏeSeJdÿœ�d�d„ƒZke=�dÐeSeJdÿœ�d�d„ƒZle=�dÑeSeJdÿœ�d�d„ƒZme=�dÒeSeJdÿœ�d�d„ƒZne=�dÓeSeJdÿœ�d �d!„ƒZoe=�dÔeSeJdÿœ�d"�d#„ƒZpe=�dÕeSeJdÿœ�d$�d%„ƒZqe=�dÖeSeJdÿœ�d&�d'„ƒZre=�d×eSeJdÿœ�d(�d)„ƒZse=�dØeSeJdÿœ�d*�d+„ƒZte=�dÙeSeJdÿœ�d,�d-„ƒZue=�dÚeSeJdÿœ�d.�d/„ƒZve=�dÛeSeJdÿœ�d0�d1„ƒZwe=�dÜeSeJdÿœ�d2�d3„ƒZxe=�dÝeSeJdÿœ�d4�d5„ƒZye=�dÞeSeJdÿœ�d6�d7„ƒZze=�dßeSeJdÿœ�d8�d9„ƒZ{e=�dàeSeJdÿœ�d:�d;„ƒZ|e=�dáeSeJdÿœ�d<�d=„ƒZ}e=�dâeSeJdÿœ�d>�d?„ƒZ~e=�dãeSeJdÿœ�d@�dA„ƒZe=�däeSeJdÿœ�dB�dC„ƒZ€e=�dåeSeJdÿœ�dD�dE„ƒZ�e=�dæeSeJdÿœ�dF�dG„ƒZ‚e=�dçeSeJdÿœ�dH�dI„ƒZƒe=�dèeSeJdÿœ�dJ�dK„ƒZ„e=�déeSeJdÿœ�dL�dM„ƒZ…e=�dêeSeJdÿœ�dN�dO„ƒZ†e=�dëeSeJdÿœ�dP�dQ„ƒZ‡e=�dìeSeJdÿœ�dR�dS„ƒZˆe=�díeSeJdÿœ�dT�dU„ƒZ‰e=�dîeSeJdÿœ�dV�dW„ƒZŠe=�dïeSeJdÿœ�dX�dY„ƒZ‹e=�dðeSeJdÿœ�dZ�d[„ƒZŒe=�dñeSeJdÿœ�d\�d]„ƒZ�e=�dòeSeJdÿœ�d^�d_„ƒZŽe=�dóeSeJdÿœ�d`�da„ƒZ�e=�dôeSeJdÿœ�db�dc„ƒZ�e=�dõeSeJdÿœ�dd�de„ƒZ‘e=�döeSeJdÿœ�df�dg„ƒZ’e=�d÷eSeJdÿœ�dh�di„ƒZ“e=�døeSeJdÿœ�dj�dk„ƒZ”e=�dùeSeJdÿœ�dl�dm„ƒZ•e=�dúeSeJdÿœ�dn�do„ƒZ–e=�dûeSeJdÿœ�dp�dq„ƒZ—e=�düeSeJdÿœ�dr�ds„ƒZ˜e=�dýeSeJdÿœ�dt�du„ƒZ™e=�dþeSeJdÿœ�dv�dw„ƒZše=�dÿeSeJdÿœ�dx�dy„ƒZ›e=�d eSeJdÿœ�dz�d{„ƒZœe=�deSeJdÿœ�d|�d}„ƒZ�e=�deSeJdÿœ�d~�d„ƒZže=�deSeJdÿœ�d€�d�„ƒZŸe=�deSeJdÿœ�d‚�dƒ„ƒZ e=�deSeJdÿœ�d„�d…„ƒZ¡e=�deSeJdÿœ�d†�d‡„ƒZ¢e=�deSeJdÿœ�dˆ�d‰„ƒZ£e=�deSeJdÿœ�dŠ�d‹„ƒZ¤e=�d	eSeJdÿœ�dŒ�d�„ƒZ¥e=�d
eSeJdÿœ�dŽ�d�„ƒZ¦e=�deSeJdÿœ�d��d‘„ƒZ§e=�deSeJdÿœ�d’�d“„ƒZ¨e=�deSeJdÿœ�d”�d•„ƒZ©e=�deSeJdÿœ�d–�d—„ƒZªe=�deSeJdÿœ�d˜�d™„ƒZ«e=�deSeJdÿœ�dš�d›„ƒZ¬e=�deSeJdÿœ�dœ�d�„ƒZ­e=�deSeJdÿœ�dž�dŸ„ƒZ®e=�deSeJdÿœ�d �d¡„ƒZ¯e=�deSeJdÿœ�d¢�d£„ƒZ°e>eA�d¤�d¥�d¦�d§�d¨�d©�dª�d«�d¬�d­�d®�d¯�d°�d±�d²�d³�d´�dµ�d¶�d·�d¸œƒ dS (  aß   Vision Transformer (ViT) in PyTorch

A PyTorch implement of Vision Transformers as described in:

'An Image Is Worth 16 x 16 Words: Transformers for Image Recognition at Scale'
    - https://arxiv.org/abs/2010.11929

`How to train your ViT? Data, Augmentation, and Regularization in Vision Transformers`
    - https://arxiv.org/abs/2106.10270

`FlexiViT: One Model for All Patch Sizes`
    - https://arxiv.org/abs/2212.08013

The official jax code is released and available at
  * https://github.com/google-research/vision_transformer
  * https://github.com/google-research/big_vision

Acknowledgments:
  * The paper authors for releasing code and weights, thanks!
  * I fixed my class token impl based on Phil Wang's https://github.com/lucidrains/vit-pytorch
  * Simple transformer style inspired by Andrej Karpathy's https://github.com/karpathy/minGPT
  * Bert reference code checks against Huggingface Transformers and Tensorflow Bert

Hacked together by / Copyright 2020, Ross Wightman
é    N)ÚOrderedDict)Úpartial)
ÚAnyÚCallableÚDictÚOptionalÚSequenceÚSetÚTupleÚTypeÚUnionÚList)ÚLiteral)ÚFinal)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STDÚIMAGENET_INCEPTION_MEANÚIMAGENET_INCEPTION_STDÚOPENAI_CLIP_MEANÚOPENAI_CLIP_STD)Ú
PatchEmbedÚMlpÚDropPathÚAttentionPoolLatentÚRmsNormÚPatchDropoutÚSwiGLUPackedÚtrunc_normal_Úlecun_normal_Úresample_patch_embedÚresample_abs_pos_embedÚuse_fused_attnÚget_act_layerÚget_norm_layerÚ	LayerTypeé   )Úbuild_model_with_cfg)Únamed_applyÚcheckpoint_seqÚadapt_input_conv)Úgenerate_default_cfgsÚregister_modelÚregister_model_deprecationsÚVisionTransformerc                
       sd   e Zd ZU ee ed< dddddejfeeeee	e	ej
ddœ‡ fdd„Zejejd	œd
d„Z‡  ZS )Ú	AttentionÚ
fused_attné   Fç        N)ÚdimÚ	num_headsÚqkv_biasÚqk_normÚ	attn_dropÚ	proj_dropÚ
norm_layerÚreturnc                    s²   t ƒ  ¡  || dkstdƒ‚|| _|| | _| jd | _tƒ | _tj	||d |d�| _
|rf|| jƒnt ¡ | _|r~|| jƒnt ¡ | _t |¡| _t 	||¡| _t |¡| _d S )Nr   ú$dim should be divisible by num_headsç      à¿é   ©Úbias)ÚsuperÚ__init__ÚAssertionErrorr3   Úhead_dimÚscaler!   r/   ÚnnÚLinearÚqkvÚIdentityÚq_normÚk_normÚDropoutr6   Úprojr7   )Úselfr2   r3   r4   r5   r6   r7   r8   ©Ú	__class__© ú]/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/vision_transformer.pyr@   <   s    


zAttention.__init__©Úxr9   c           
      C   sê   |j \}}}|  |¡ ||d| j| j¡ ddddd¡}| d¡\}}}|  |¡|  |¡ }}| j	r„t
j|||| jrz| jjndd�}n8|| j }|| dd	¡ }	|	jd	d
�}	|  |	¡}	|	| }| dd¡ |||¡}|  |¡}|  |¡}|S )Nr<   é   r   r%   é   r1   ©Z	dropout_péþÿÿÿéÿÿÿÿ©r2   )ÚshaperF   Úreshaper3   rB   ÚpermuteZunbindrH   rI   r/   ÚFÚscaled_dot_product_attentionÚtrainingr6   ÚprC   Ú	transposeÚsoftmaxrK   r7   )
rL   rR   ÚBÚNÚCrF   ÚqÚkÚvÚattnrO   rO   rP   ÚforwardT   s(    *  þ



zAttention.forward)Ú__name__Ú
__module__Ú__qualname__r   ÚboolÚ__annotations__rD   Ú	LayerNormÚintÚfloatÚModuler@   ÚtorchÚTensorri   Ú__classcell__rO   rO   rM   rP   r.   9   s$   
ø÷r.   c                       s>   e Zd Zd
eeeddœ‡ fdd„Zejejdœdd	„Z	‡  Z
S )Ú
LayerScaleçñhãˆµøä>FN)r2   Úinit_valuesÚinplacer9   c                    s*   t ƒ  ¡  || _t |t |¡ ¡| _d S ©N)r?   r@   ry   rD   Ú	Parameterrs   ZonesÚgamma)rL   r2   rx   ry   rM   rO   rP   r@   m   s    
zLayerScale.__init__rQ   c                 C   s   | j r| | j¡S || j S rz   )ry   Zmul_r|   ©rL   rR   rO   rO   rP   ri   w   s    zLayerScale.forward)rw   F)rj   rk   rl   rp   rq   rm   r@   rs   rt   ri   ru   rO   rO   rM   rP   rv   l   s     üû
rv   c                       sr   e Zd Zdddddddejejef
eeee	e	eee
e eejejejddœ‡ fdd„Zejejdœd	d
„Z‡  ZS )ÚBlockç      @Fr1   N©r2   r3   Ú	mlp_ratior4   r5   r7   r6   rx   Ú	drop_pathÚ	act_layerr8   Ú	mlp_layerr9   c              	      s¼   t ƒ  ¡  ||ƒ| _t|||||||d�| _|r<t||d�nt ¡ | _|	dkrVt	|	ƒnt ¡ | _
||ƒ| _||t|| ƒ|
|d�| _|r”t||d�nt ¡ | _|	dkr®t	|	ƒnt ¡ | _d S )N©r3   r4   r5   r6   r7   r8   ©rx   r1   ©Zin_featuresÚhidden_featuresrƒ   Údrop)r?   r@   Únorm1r.   rh   rv   rD   rG   Úls1r   Ú
drop_path1Únorm2rp   ÚmlpÚls2Ú
drop_path2©rL   r2   r3   r�   r4   r5   r7   r6   rx   r‚   rƒ   r8   r„   rM   rO   rP   r@   |   s,    

ù	

üzBlock.__init__rQ   c              
   C   sD   ||   |  |  |  |¡¡¡¡ }||  |  |  |  |¡¡¡¡ }|S rz   )rŒ   r‹   rh   rŠ   r�   r�   rŽ   r�   r}   rO   rO   rP   ri   £   s      zBlock.forward)rj   rk   rl   rD   ÚGELUro   r   rp   rq   rm   r   rr   r@   rs   rt   ri   ru   rO   rO   rM   rP   r~   {   s4   óò'r~   c                       s€   e Zd Zdddddddejejef
eeee	e	eee
e eejejejddœ‡ fdd„Zddœd	d
„Zejejdœdd„Z‡  ZS )ÚResPostBlockr   Fr1   Nr€   c              	      s–   t ƒ  ¡  || _t|||||||d�| _||ƒ| _|	dkrBt|	ƒnt ¡ | _	||t
|| ƒ|
|d�| _||ƒ| _|	dkr€t|	ƒnt ¡ | _|  ¡  d S )Nr…   r1   r‡   )r?   r@   rx   r.   rh   rŠ   r   rD   rG   rŒ   rp   rŽ   r�   r�   Úinit_weightsr‘   rM   rO   rP   r@   ª   s,    
ù	

ü
zResPostBlock.__init__©r9   c                 C   s6   | j d k	r2tj | jj| j ¡ tj | jj| j ¡ d S rz   )rx   rD   ÚinitÚ	constant_rŠ   Úweightr�   ©rL   rO   rO   rP   r”   Ó   s    
zResPostBlock.init_weightsrQ   c                 C   s8   ||   |  |  |¡¡¡ }||  |  |  |¡¡¡ }|S rz   )rŒ   rŠ   rh   r�   r�   rŽ   r}   rO   rO   rP   ri   Ù   s    zResPostBlock.forward)rj   rk   rl   rD   r’   ro   r   rp   rq   rm   r   rr   r@   r”   rs   rt   ri   ru   rO   rO   rM   rP   r“   ©   s6   óò)r“   c                       sˆ   e Zd ZU dZee ed< dddddddejej	df
e
e
eeeeeee eejejeej ddœ‡ fdd	„Zejejd
œdd„Z‡  ZS )ÚParallelScalingBlockz¤ Parallel ViT block (MLP & Attention in parallel)
    Based on:
      'Scaling Vision Transformers to 22 Billion Parameters` - https://arxiv.org/abs/2302.05442
    r/   r   Fr1   Nr€   c                    s~  t ƒ  ¡  || dkstdƒ‚|| _|| | _| jd | _tƒ | _t|| ƒ}|d|  }||ƒ| _	t
j|||d�| _|g|gd  | _|r¦|  dd ¡ |  dd ¡ n,| jdt d| ¡dd	� t
 t |¡¡| _|rà|| jƒnt
 ¡ | _|rø|| jƒnt
 ¡ | _t
 |¡| _t
 ||¡| _t
 |¡| _|
ƒ | _t
 ||¡| _|d k	�rTt||d
�nt
 ¡ | _|	dk�rpt|	ƒnt
 ¡ | _ d S )Nr   r:   r;   r<   r=   r4   Úmlp_biasF)Ú
persistentr†   r1   )!r?   r@   rA   r3   rB   rC   r!   r/   rp   Úin_normrD   rE   Úin_projÚin_splitZregister_bufferZregister_parameterrs   Úzerosr{   r›   rG   rH   rI   rJ   r6   Úattn_out_projÚmlp_dropÚmlp_actÚmlp_out_projrv   Úlsr   r‚   )rL   r2   r3   r�   r4   r5   r7   r6   rx   r‚   rƒ   r8   r„   Zmlp_hidden_dimZin_proj_out_dimrM   rO   rP   r@   æ   s2    


 zParallelScalingBlock.__init__rQ   c                 C   s„  |j \}}}|  |¡}| jd k	rBt || jjt | j	| jf¡¡}n
|  |¡}tj
|| jdd�\}}}}	|  | ||| j| j¡¡ dd¡}|  | ||| j| j¡¡ dd¡}|	 ||| j| j¡ dd¡}	| jrîtj|||	| jrä| jjndd�}
n8|| j }|| dd¡ }|jdd�}|  |¡}||	 }
|
 dd¡ |||¡}
|  |
¡}
|  |¡}|  |¡}|  |¡}|  |  |
| ¡¡}|| }|S )NrW   rX   r%   rS   r1   rU   rV   ) rY   r�   r›   r\   Zlinearrž   r˜   rs   Úcatr4   ÚsplitrŸ   rH   Úviewr3   rB   r`   rI   r/   r]   r^   r6   r_   rC   ra   rZ   r¡   r£   r¢   r¤   r‚   r¥   )rL   rR   rb   rc   rd   ÚyZx_mlpre   rf   rg   Zx_attnrh   rO   rO   rP   ri     s:    

"
""  þ





zParallelScalingBlock.forward)rj   rk   rl   Ú__doc__r   rm   rn   rD   r’   ro   rp   rq   r   rr   r@   rs   rt   ri   ru   rO   rO   rM   rP   rš   ß   s8   
óò.rš   c                       sª   e Zd ZdZddddddddejejefeeee	e
e
ee	 e	e	e	ejejejddœ‡ fdd	„Zejejd
œdd„Zejjejejd
œdd„ƒZejejd
œdd„Z‡  ZS )ÚParallelThingsBlockz¿ Parallel ViT block (N parallel attention followed by N parallel MLP)
    Based on:
      `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    rS   r   FNr1   )r2   r3   Únum_parallelr�   r4   r5   rx   r7   r6   r‚   rƒ   r8   r„   r9   c                    s  t ƒ  ¡  || _t ¡ | _t ¡ | _t|ƒD ]Þ}| j t 	t
d||ƒfdt|||||	||d�fd|rpt||d�nt ¡ fd|
dkrŠt|
ƒnt ¡ fgƒ¡¡ | j t 	t
d||ƒfd||t|| ƒ||d	�fd|rÞt||d�nt ¡ fd|
dkrøt|
ƒnt ¡ fgƒ¡¡ q,d S )
NÚnormrh   r…   r¥   r†   r‚   r1   rŽ   )rˆ   rƒ   r‰   )r?   r@   r¬   rD   Z
ModuleListÚattnsÚffnsÚrangeÚappendÚ
Sequentialr   r.   rv   rG   r   rp   )rL   r2   r3   r¬   r�   r4   r5   rx   r7   r6   r‚   rƒ   r8   r„   Ú_rM   rO   rP   r@   C  s<    



ù	ô


ü÷zParallelThingsBlock.__init__rQ   c                    sP   ˆ t  ‡ fdd„| jD ƒ¡jdd� ‰ ˆ t  ‡ fdd„| jD ƒ¡jdd� ‰ ˆ S )Nc                    s   g | ]}|ˆ ƒ‘qS rO   rO   ©Ú.0rh   ©rR   rO   rP   Ú
<listcomp>s  s     z4ParallelThingsBlock._forward_jit.<locals>.<listcomp>r   rX   c                    s   g | ]}|ˆ ƒ‘qS rO   rO   ©rµ   Zffnr¶   rO   rP   r·   t  s     )rs   Ústackr®   Úsumr¯   r}   rO   r¶   rP   Ú_forward_jitr  s    &&z ParallelThingsBlock._forward_jitc                    s<   ˆ t ‡ fdd„| jD ƒƒ ‰ ˆ t ‡ fdd„| jD ƒƒ ‰ ˆ S )Nc                 3   s   | ]}|ˆ ƒV  qd S rz   rO   r´   r¶   rO   rP   Ú	<genexpr>y  s     z/ParallelThingsBlock._forward.<locals>.<genexpr>c                 3   s   | ]}|ˆ ƒV  qd S rz   rO   r¸   r¶   rO   rP   r¼   z  s     )rº   r®   r¯   r}   rO   r¶   rP   Ú_forwardw  s    zParallelThingsBlock._forwardc                 C   s,   t j ¡ st j ¡ r|  |¡S |  |¡S d S rz   )rs   ÚjitÚis_scriptingÚ
is_tracingr»   r½   r}   rO   rO   rP   ri   }  s    
zParallelThingsBlock.forward)rj   rk   rl   rª   rD   r’   ro   r   rp   rq   rm   r   rr   r@   rs   rt   r»   r¾   Úignorer½   ri   ru   rO   rO   rM   rP   r«   >  s@   òñ/r«   c                #       sh  e Zd ZU dZee ed< ddddddd	d	d
ddddddddddddddddddeddee	f e
eeeef f e
eeeef f eeed eeeeeeee eeeeee eeeeeeeeed eeee ee eej eej ddœ!‡ fdd„Zdd„ Zd@eddœdd„Zejddœdd„Zej ¡ dAeeddœdd „ƒZejjed!œd"d#„ƒZejjdBee d$œd%d&„ƒZ!ejjdCedd'œd(d)„ƒZ"ejjejd!œd*d+„ƒZ#dDedd,œd-d.„Z$ej%ej%d/œd0d1„Z&dEej%e
ee'f e(ej% d3œd4d5„Z)dFej%e
ee'f eeeee
ej%eej% f  d6œd7d8„Z*ej%ej%d/œd9d:„Z+dGej%eej%d;œd<d=„Z,ej%ej%d/œd>d?„Z-‡  Z.S )Hr-   z© Vision Transformer

    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
        - https://arxiv.org/abs/2010.11929
    Údynamic_img_sizeéà   é   r<   éè  Útokené   é   r   TFNr   r1   Ú ©rÉ   ÚavgrÆ   Úmap)ÚskipÚjaxÚjax_nlhbÚmocorÉ   )!Úimg_sizeÚ
patch_sizeÚin_chansÚnum_classesÚglobal_poolÚ	embed_dimÚdepthr3   r�   r4   r5   rx   Úclass_tokenÚno_embed_classÚ
reg_tokensÚpre_normÚfc_normrÂ   Údynamic_img_padÚ	drop_rateÚpos_drop_rateÚpatch_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rateÚweight_initÚfix_initÚembed_layerr8   rƒ   Úblock_fnr„   r9   c!           %         s¤  t ƒ  ¡  |dkst‚|s&|dks&t‚|dkr6|dkn|}!tˆƒpNttjdd�‰tˆ ƒp\tj‰ || _	|| _
ˆ | _| _|r~dnd| _|  j|7  _|| _|| _|| _|| _d	| _i }"|rÊ|" td	d
d�¡ |f |||ˆ| |dœ|"—Ž| _| jj}#|�rt t ddˆ¡¡nd| _|�r,t t d|ˆ¡¡nd| _|�r<|#n|#| j }$t t d|$ˆ¡d ¡| _tj|d�| _|dk�rŠt || jd�| _!n
t "¡ | _!|�r¢ˆˆƒnt "¡ | _#dd„ t $d||¡D ƒ‰tj%‡ ‡‡‡‡‡‡‡‡‡	‡
‡‡fdd„t&|ƒD ƒŽ | _'|!�sˆˆƒnt "¡ | _(|dk�r2t)| jˆ	ˆˆd�| _*nd| _*|!�rFˆˆƒnt "¡ | _+t |¡| _,|dk�rtt -| j|¡nt "¡ | _.|dk�r’|  /|¡ |�r |  0¡  dS )aÍ  
        Args:
            img_size: Input image size.
            patch_size: Patch size.
            in_chans: Number of image input channels.
            num_classes: Mumber of classes for classification head.
            global_pool: Type of global pooling for final sequence (default: 'token').
            embed_dim: Transformer embedding dimension.
            depth: Depth of transformer.
            num_heads: Number of attention heads.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            qkv_bias: Enable bias for qkv projections if True.
            init_values: Layer-scale init values (layer-scale enabled if not None).
            class_token: Use class token.
            no_embed_class: Don't include position embeddings for class (or reg) tokens.
            reg_tokens: Number of register tokens.
            fc_norm: Pre head norm after pool (instead of before), if None, enabled when global_pool == 'avg'.
            drop_rate: Head dropout rate.
            pos_drop_rate: Position embedding dropout rate.
            attn_drop_rate: Attention dropout rate.
            drop_path_rate: Stochastic depth rate.
            weight_init: Weight initialization scheme.
            fix_init: Apply weight initialization fix (scaling w/ layer index).
            embed_layer: Patch embedding layer.
            norm_layer: Normalization layer.
            act_layer: MLP activation layer.
            block_fn: Transformer block layer.
        rÊ   rÆ   NrË   ç�íµ ÷Æ°>)Úepsr%   r   FZNHWC)Zstrict_img_sizeZ
output_fmt)rÑ   rÒ   rÓ   rÖ   r>   rÝ   ç{®Gáz”?)r_   ©Únum_prefix_tokensc                 S   s   g | ]}|  ¡ ‘qS rO   )Úitem)rµ   rR   rO   rO   rP   r·   ú  s     z.VisionTransformer.__init__.<locals>.<listcomp>c                    s0   g | ](}ˆˆˆ	ˆˆˆˆˆ
ˆˆ| ˆˆ ˆd �‘qS ))r2   r3   r�   r4   r5   rx   r7   r6   r‚   r8   rƒ   r„   rO   )rµ   Úi©rƒ   râ   rç   ZdprrÖ   rx   r„   r�   r8   r3   rá   r5   r4   rO   rP   r·   û  s   òôrÌ   )r3   r�   r8   rÍ   )1r?   r@   rA   r#   r   rD   ro   r"   r’   rÔ   rÕ   Znum_featuresrÖ   rì   Znum_reg_tokensZhas_class_tokenrÙ   rÂ   Úgrad_checkpointingÚupdateÚdictÚpatch_embedÚnum_patchesr{   rs   r    Ú	cls_tokenÚ	reg_tokenZrandnÚ	pos_embedrJ   Úpos_dropr   Ú
patch_droprG   Únorm_preZlinspacer²   r°   Úblocksr­   r   Ú	attn_poolrÜ   Ú	head_droprE   Úheadr”   Úfix_init_weight)%rL   rÑ   rÒ   rÓ   rÔ   rÕ   rÖ   r×   r3   r�   r4   r5   rx   rØ   rÙ   rÚ   rÛ   rÜ   rÂ   rÝ   rÞ   rß   rà   rá   râ   rã   rä   rå   ræ   r8   rƒ   rç   r„   Zuse_fc_normZ
embed_argsrô   Z	embed_lenrM   rï   rP   r@   Œ  s|    ?
úù	  
þ

&ñ

ü
"

zVisionTransformer.__init__c                 C   sL   dd„ }t | jƒD ]4\}}||jjjj|d ƒ ||jjjj|d ƒ qd S )Nc                 S   s   |   t d| ¡¡ d S )Ng       @)Zdiv_ÚmathÚsqrt)ÚparamZ	_layer_idrO   rO   rP   Úrescale!  s    z2VisionTransformer.fix_init_weight.<locals>.rescaler%   )Ú	enumeraterû   rh   rK   r˜   ÚdatarŽ   Úfc2)rL   r  Zlayer_idÚlayerrO   rO   rP   rÿ      s    z!VisionTransformer.fix_init_weight)Úmoder9   c                 C   sd   |dkst ‚d|kr"t | j¡ nd}t| jdd� | jd k	rPtjj	| jdd� t
t||ƒ| ƒ d S )N)rÎ   rÏ   rÐ   rÉ   Znlhbr1   rê   ©Ústdrè   )rA   r   ÚlogrÔ   r   r÷   rõ   rD   r–   Únormal_r'   Úget_init_weights_vit)rL   r  Ú	head_biasrO   rO   rP   r”   (  s    
zVisionTransformer.init_weights)Úmr9   c                 C   s   t |ƒ d S rz   )Úinit_weights_vit_timm)rL   r  rO   rO   rP   Ú_init_weights0  s    zVisionTransformer._init_weights)Úcheckpoint_pathÚprefixr9   c                 C   s   t | ||ƒ d S rz   )Ú_load_weights)rL   r  r  rO   rO   rP   Úload_pretrained4  s    z!VisionTransformer.load_pretrainedr•   c                 C   s
   dddhS )Nr÷   rõ   Z
dist_tokenrO   r™   rO   rO   rP   Úno_weight_decay8  s    z!VisionTransformer.no_weight_decay)Úcoarser9   c                 C   s   t dddgd�S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )Ústemrû   )rò   )rL   r  rO   rO   rP   Úgroup_matcher<  s    þzVisionTransformer.group_matcher)Úenabler9   c                 C   s
   || _ d S rz   )rð   )rL   r  rO   rO   rP   Úset_grad_checkpointingC  s    z(VisionTransformer.set_grad_checkpointingc                 C   s   | j S rz   )rþ   r™   rO   rO   rP   Úget_classifierG  s    z VisionTransformer.get_classifier)rÔ   r9   c                 C   s|   || _ |d k	rX|dkst‚|dkr:| jd kr:dsRtdƒ‚n|dkrR| jd k	rRd | _|| _|dkrnt | j|¡nt ¡ | _d S )NrÊ   rÌ   Fz=Cannot currently add attention pooling in reset_classifier().zmap r   )	rÔ   rA   rü   rÕ   rD   rE   rÖ   rG   rþ   )rL   rÔ   rÕ   rO   rO   rP   Úreset_classifierK  s    z"VisionTransformer.reset_classifierrQ   c                 C   sì   | j rD|j\}}}}t| j||f| jr*dn| jd�}| |d|¡}n| j}g }| jd k	rt| | j 	|jd dd¡¡ | j
d k	rš| | j
 	|jd dd¡¡ | jrÂ|| }|râtj||g dd�}n |rÚtj||g dd�}|| }|  |¡S )Nr   rë   rW   r%   rX   )rÂ   rY   r    r÷   rÙ   rì   r¨   rõ   r±   Úexpandrö   rs   r¦   rø   )rL   rR   rb   ÚHÚWrd   r÷   Zto_catrO   rO   rP   Ú
_pos_embedV  s,    ý

zVisionTransformer._pos_embedr%   )rR   Únr9   c                 C   sŠ   g t | jƒ }}tt|tƒr*t|| |ƒn|ƒ}|  |¡}|  |¡}|  |¡}|  	|¡}t
| jƒD ]"\}}||ƒ}||krb| |¡ qb|S rz   )Úlenrû   ÚsetÚ
isinstancerp   r°   ró   r!  rù   rú   r  r±   )rL   rR   r"  ÚoutputsZ
num_blocksZtake_indicesrî   ZblkrO   rO   rP   Ú_intermediate_layersw  s     



z&VisionTransformer._intermediate_layers)rR   r"  rZ   Úreturn_prefix_tokensr­   r9   c                    s€   ˆ  ˆ|¡}|r"‡fdd„|D ƒ}‡fdd„|D ƒ}‡fdd„|D ƒ}|rfˆjj‰ ‡ ‡fdd„|D ƒ}|rxtt||ƒƒS t|ƒS )zs Intermediate layer accessor (NOTE: This is a WIP experiment).
        Inspired by DINO / DINOv2 interface
        c                    s   g | ]}ˆ   |¡‘qS rO   )r­   ©rµ   Úoutr™   rO   rP   r·   ™  s     z=VisionTransformer.get_intermediate_layers.<locals>.<listcomp>c                    s"   g | ]}|d d …dˆ j …f ‘qS )Nr   rë   r)  r™   rO   rP   r·   š  s     c                    s"   g | ]}|d d …ˆ j d …f ‘qS rz   rë   r)  r™   rO   rP   r·   ›  s     c                    s:   g | ]2}|  ˆjd  ˆ d  ˆ d d¡ d ddd¡ ¡ ‘qS )r   r%   rW   r<   rS   )rZ   rY   r[   Ú
contiguousr)  )Ú	grid_sizerR   rO   rP   r·   Ÿ  s   ÿ)r'  ró   r,  ÚtupleÚzip)rL   rR   r"  rZ   r(  r­   r&  Zprefix_tokensrO   )r,  rL   rR   rP   Úget_intermediate_layers‹  s    þz)VisionTransformer.get_intermediate_layersc                 C   s^   |   |¡}|  |¡}|  |¡}|  |¡}| jrFtj ¡ sFt| j	|ƒ}n
|  	|¡}|  
|¡}|S rz   )ró   r!  rù   rú   rð   rs   r¾   r¿   r(   rû   r­   r}   rO   rO   rP   Úforward_features¨  s    





z"VisionTransformer.forward_features)rR   Ú
pre_logitsr9   c                 C   s|   | j d k	r|   |¡}n@| jdkr@|d d …| jd …f jdd�}n| jrV|d d …df }|  |¡}|  |¡}|rr|S |  |¡S )NrË   r%   rX   r   )rü   rÕ   rì   ÚmeanrÜ   rý   rþ   )rL   rR   r1  rO   rO   rP   Úforward_head´  s    

 

zVisionTransformer.forward_headc                 C   s   |   |¡}|  |¡}|S rz   )r0  r3  r}   rO   rO   rP   ri   ¿  s    

zVisionTransformer.forward)rÉ   )rÉ   )F)T)N)r%   )r%   FFF)F)/rj   rk   rl   rª   r   rm   rn   r   r~   r   r   rp   r
   r   rq   r   r   r$   r   rD   rr   r@   rÿ   Ústrr”   r  rs   r¾   rÁ   r  r	   r  r   r  r  r  r  rt   r!  r   r   r'  r/  r0  r3  ri   ru   rO   rO   rM   rP   r-   „  sÔ   
ßÞ $ ý
ü    ú
ùrÉ   )ÚmoduleÚnamer9   c                 C   sJ   t | tjƒr4t| jdd� | jdk	rFtj | j¡ nt| dƒrF|  	¡  dS )zE ViT weight initialization, original timm impl (for reproducibility) rê   r	  Nr”   )
r%  rD   rE   r   r˜   r>   r–   Úzeros_Úhasattrr”   )r5  r6  rO   rO   rP   r  Å  s    

r  r1   )r5  r6  r  r9   c                 C   s¾   t | tjƒrx| d¡r6tj | j¡ tj | j|¡ qºtj 	| j¡ | jdk	rºd|krhtjj
| jdd�ntj | j¡ nBt | tjƒr¨t| jƒ | jdk	rºtj | j¡ nt| dƒrº|  ¡  dS )z5 ViT weight initialization, matching JAX (Flax) impl rþ   NrŽ   rè   r	  r”   )r%  rD   rE   Ú
startswithr–   r7  r˜   r—   r>   Úxavier_uniform_r  ZConv2dr   r8  r”   )r5  r6  r  rO   rO   rP   Úinit_weights_vit_jaxÏ  s    

*


r;  c                 C   s’   t | tjƒr|d|krTt dt| jjd d | jjd  ƒ ¡}tj 	| j| |¡ ntj 
| j¡ | jdk	rŽtj | j¡ nt| dƒrŽ|  ¡  dS )zI ViT weight initialization, matching moco-v3 impl minus fixed PatchEmbed rF   g      @r   r<   r%   Nr”   )r%  rD   rE   r   r  rq   r˜   rY   r–   Zuniform_r:  r>   r7  r8  r”   )r5  r6  ÚvalrO   rO   rP   Úinit_weights_vit_mocoá  s    *

r=  rÎ   )r  r  r9   c                 C   s(   d| krt t|d�S d| kr tS tS d S )NrÎ   )r  rÐ   )r   r;  r=  r  )r  r  rO   rO   rP   r  ð  s
    r  rO   ÚbicubicF)ÚposembÚ
posemb_newrì   Úgs_newÚinterpolationÚ	antialiasr9   c           
      C   s2  |j d }|r>| dd…d|…f | d|d…f  }}||8 }n| dd…dd…f | d  }}tt t|ƒ¡ƒ}	t|ƒsŠtt |¡ƒgd }t|ƒdksšt‚t d| j › d|	|	g› d|j › d|› d�	¡ | d|	|	d	¡ 	dd
dd¡}t
j||||dd�}| 	ddd
d¡ d|d |d  d	¡}tj||gdd�} | S )a=   Rescale the grid of position embeddings when loading from state_dict.

    *DEPRECATED* This function is being deprecated in favour of resample_abs_pos_embed

    Adapted from:
        https://github.com/google-research/vision_transformer/blob/00883dd691c63a6830751563748663526e811cee/vit_jax/checkpoint.py#L224
    r%   Nr   rS   zResized position embedding: z (z) to z).rW   r<   F)Úsizer  rC  Zalign_cornersrX   )rY   rp   r   r  r#  rA   Ú_loggerÚinforZ   r[   r\   Zinterpolaters   r¦   )
r?  r@  rì   rA  rB  rC  Zntok_newZposemb_prefixZposemb_gridZgs_oldrO   rO   rP   Úresize_pos_embedù  s    
&
,&rG  )Úmodelr  r  r9   c                    s$  ddl }dKdd„‰ | |¡‰d}d}d}|s\dˆkr:d	}n"d
ˆkrLd}d}ndˆkr\d}d}t| jdƒ�rj| jj}t|dƒ }|r†|n|j}	|	jj t	|	jjj
d ˆ ˆ|› d� ƒƒ¡ |	jj ˆ ˆ|› d� ƒ¡ |	jj ˆ ˆ|› d� ƒ¡ |�sVt|jƒD �]R\}
}t|jƒD �]:\}}|› d|
d › d|d › d�}tdƒD ]¢}t|d|d › �ƒj ˆ ˆ|› d|d › d� ƒ¡ t|d|d › �ƒj ˆ ˆ|› d|d › d� ƒ¡ t|d|d › �ƒj ˆ ˆ|› d|d › d� ƒ¡ �qD|jdk	�r|jjj ˆ ˆ|› d� ƒ¡ |jjj ˆ ˆ|› d� ƒ¡ |jjj ˆ ˆ|› d � ƒ¡ �q�q ˆ ˆ|› d!� ƒ}n$t	| jjjj
d ˆ ˆ|› d!� ƒƒ}|j
d"d… | jjjj
d"d… k�rÔt|| jjjj
d"d… ||dd#�}| jjj |¡ | jjj ˆ ˆ|› d$� ƒ¡ | jdk	�r,| j ˆ ˆ|› d%� dd&�¡ |�rJˆ ˆ|› d'� dd&�}nˆ ˆ|› d(� dd&�}|j
| jj
k�r¬|j
}t| d)dƒ�rˆdn
t| d*dƒ}t|| jj|||dd+�}| j |¡ | jj ˆ ˆ|› d,� ƒ¡ | jj ˆ ˆ|› d-� ƒ¡ t| jtjƒ�rl|› d.�ˆk�rl| jjj
d ˆ|› d.� j
d/ k�rl| jj ˆ ˆ|› d0� ƒ¡ | jj ˆ ˆ|› d.� ƒ¡ | jdk	�rB|› d1�}|d2 ‰| jj ˆ ˆ|› d3� dd&�¡ | jjj t  !‡ ‡‡fd4d5„d6D ƒ¡¡ | jjj t  !‡ ‡‡fd7d5„d6D ƒ¡¡ | jj"j ˆ ˆˆ› d8� dd&� #d¡j$¡ | jj"j ˆ ˆˆ› d9� dd&� %d/¡¡ | jjj ˆ ˆˆ› d:� ƒ #d¡¡ | jjj ˆ ˆˆ› d;� ƒ¡ | jjj ˆ ˆ|› d<� ƒ¡ | jjj ˆ ˆ|› d=� ƒ¡ td>ƒD ]n}t| jj&d?|d › �ƒj ˆ ˆ|› d@|› d� ƒ¡ t| jj&d?|d › �ƒj ˆ ˆ|› d@|› d� ƒ¡ �qÒ|�rLdAndB\}}}t| j '¡ ƒD �]¸\}
}|› dC|
› d�}|dD|› d� ‰|j(j ˆ ˆ|› d<� ƒ¡ |j(j ˆ ˆ|› d=� ƒ¡ |j)j*j t  !‡ ‡‡fdEd5„dFD ƒ¡¡ |j)j*j t  !‡ ‡‡fdGd5„dFD ƒ¡¡ |j)jj ˆ ˆˆ› d:� ƒ #d¡¡ |j)jj ˆ ˆˆ› d;� ƒ¡ |j+j ˆ ˆ|› dH|› d� ƒ¡ |j+j ˆ ˆ|› dH|› d� ƒ¡ td>ƒD ]v}t|j&d?|d › �ƒj ˆ ˆ|› dI|› dJ|› d� ƒ¡ t|j&d?|d › �ƒj ˆ ˆ|› dI|› dJ|› d� ƒ¡ �q¤�qddS )LzV Load weights from .npz checkpoints for official Google Brain Flax implementation
    r   NTc                 S   s¦   | j dkrF| jd | jd   kr:| jd   kr:dkrFn n|  ¡ } |rœ| j dkrh|  ddddg¡} n4| j dkr„|  dddg¡} n| j dkrœ|  ddg¡} t | ¡S )NrT   r   r%   rS   r<   )ÚndimrY   Úflattenr`   rs   Z
from_numpy)ÚwÚtrO   rO   rP   Ú_n2p   s    >


z_load_weights.<locals>._n2pÚbilinearFzopt/target/embedding/kernelzopt/target/zparams/embedding/kernelzparams/zparams/img/embedding/kernelzparams/img/Úbackboner  r%   zconv_root/kernelzgn_root/scalezgn_root/biasÚblockz/unitú/r<   Úconvú/kernelr­   Zgnz/scaleú/biaszconv_proj/kernelzgn_proj/scalezgn_proj/biaszembedding/kernelrV   ©rB  rC  Úverbosezembedding/biasÚcls©rL  Zpos_embeddingz(Transformer/posembed_input/pos_embeddingrÙ   rì   ©Únew_sizerì   rB  rC  rV  zTransformer/encoder_norm/scalezTransformer/encoder_norm/biasz	head/biasrW   zhead/kernelz
MAPHead_0/zMultiHeadDotProductAttention_0/Zprobec                    s.   g | ]&}ˆ ˆˆ› |› d � dd�  d¡j‘qS ©rS  FrX  r%   ©rJ  ÚT©rµ   r"  ©rM  Z
mha_prefixrK  rO   rP   r·   ~  s    z!_load_weights.<locals>.<listcomp>)ÚkeyÚvaluec                    s,   g | ]$}ˆ ˆˆ› |› d � dd�  d¡‘qS ©rT  FrX  rW   ©rZ   r^  r_  rO   rP   r·   €  s    zquery/kernelz
query/biasz
out/kernelzout/biaszLayerNorm_0/scalezLayerNorm_0/biasrS   ÚfczMlpBlock_0/Dense_)r   r   r%   )r%   r<   rS   zTransformer/encoderblock_ZMultiHeadDotProductAttention_c                    s.   g | ]&}ˆ ˆˆ› |› d � dd�  d¡j‘qS r[  r\  r^  r_  rO   rP   r·   ’  s    )Úqueryr`  ra  c                    s,   g | ]$}ˆ ˆˆ› |› d � dd�  d¡‘qS rb  rc  r^  r_  rO   rP   r·   ”  s    Z
LayerNorm_Z	MlpBlock_z/Dense_)T),ÚnumpyÚloadr8  ró   rO  r  rR  r˜   Zcopy_r)   rY   r­   r>   r  Zstagesrû   r°   ÚgetattrZ
downsamplerK   r   rõ   r÷   r    r,  r%  rþ   rD   rE   rü   ZlatentÚkvrs   r¦   re   rJ  r]  rZ   rŽ   ÚchildrenrŠ   rh   rF   r�   )rH  r  r  ÚnprB  rC  Z
big_visionrO  Z	stem_onlyr  rî   ZstageÚjrP  ÚbpÚrZembed_conv_wZpos_embed_wZ	old_shaperì   Zblock_prefixZmha_subZb_subZln1_subrO   r_  rP   r    sâ    

,448& ÿ$ûúÿ þ
 ÿ
ÿ
*($48ÿ
ÿ
$""8r  úvisual.)Ú
state_dictrH  r  r9   c              
   C   s  i }ddddddddd	d
g
}|   ¡ D ]è\}}| |¡s8q$| |d¡}|D ]}| |d |d ¡}qH|dkr�d}| dd¡}t |jd ¡|d< nt|dkr®d}| d¡ d¡}nV|dk�r| d¡}|jd |jjd k�rt	||jt
|dƒrðdn
t
|ddƒ|jjƒ}|||< q$|S )N)Zconv1úpatch_embed.proj)Zpositional_embeddingr÷   )ztransformer.resblocks.zblocks.)Zln_prerú   )Zln_postr­   )Zln_r­   )Zin_proj_zqkv.)Zout_projrK   )zmlp.c_fczmlp.fc1)z
mlp.c_projzmlp.fc2rÉ   r   r%   rK   zhead.weightz	head.biasZclass_embeddingrõ   r÷   rÙ   rì   )Úitemsr9  Úreplacer`   rs   r    rY   Z	unsqueezer÷   rG  rh  ró   r,  )rp  rH  r  Úout_dictZswapsrf   rg   ÚsprO   rO   rP   Ú_convert_openai_clipŸ  sH    ö


ü
rv  )rp  rH  r9   c                 C   sÊ   dd l }i }|  dd ¡ d| krn|  d¡|d< |  d¡| d d d …df  |d< |  d¡d d …dd …f |d< |  ¡ D ]N\}}| d|¡rž||| d	d
¡< qvn| d|¡r¼||| dd¡< qv|||< qv|S )Nr   Ú
mask_tokenZregister_tokensrö   rõ   r÷   r%   z(blocks\.(\d+)\.mlp\.w12\.(?:weight|bias)Zw12Zfc1z'blocks\.(\d+)\.mlp\.w3\.(?:weight|bias)Zw3r  )ÚreÚpoprr  Úmatchrs  )rp  rH  rx  rt  rf   rg   rO   rO   rP   Ú_convert_dinov2Í  s     "
r{  T)rp  rH  Úadapt_layer_scalerB  rC  r9   c              	      sÌ  ddl }i }|  d| ¡} |  d| ¡} d‰ d| kr:t| |ƒS d| krPt| |dd	�S d
| krbt| |ƒ} d| krv| d } d‰ d| kr‚d‰ ˆ rœ‡ fdd„|  ¡ D ƒ} |  ¡ D �] \}}d|k�r8|jjjj\}	}
}}t	|jƒdk �r |jjjj\}	}
}}| 
|	d||¡}|jd |k�s |jd |k�r¾t|||f||dd�}n†|dk�r’|jd |jjd k�r’t|ddƒ�rldn
t|ddƒ}t||jj|||dd�}n,|�r²d|k�r²| dd|¡}nd |k�r¾q¤|||< q¤|S )!zJ convert patch embedding weight from manual patchify + linear proj to convr   NrH  rp  rÉ   zvisual.class_embeddingzmodule.visual.class_embeddingzmodule.visual.©r  rw  Úencoderzmodule.zvisual.trunk.pos_embedzvisual.trunk.c                    s,   i | ]$\}}|  ˆ ¡r|tˆ ƒd … |“qS rz   )r9  r#  )rµ   rf   rg   r}  rO   rP   Ú
<dictcomp>  s     
  z(checkpoint_filter_fn.<locals>.<dictcomp>zpatch_embed.proj.weightrT   rW   rV   TrU  r÷   r%   rÙ   Frì   rY  Zgamma_zgamma_([0-9])z
ls\1.gammar1  )rx  Úgetrv  r{  rr  ró   rK   r˜   rY   r#  rZ   r   r÷   rh  r    r,  Úsub)rp  rH  r|  rB  rC  rx  rt  rf   rg   ÚOÚIr  r   rì   rO   r}  rP   Úcheckpoint_filter_fnä  s`    


 û"ú

r„  )Úurlr9   c                 K   s    | ddd dddt tdddœ|–S )	NrÅ   )r<   rÃ   rÃ   gÍÌÌÌÌÌì?r>  Trq  rþ   )r…  rÔ   Ú
input_sizeZ	pool_sizeÚcrop_pctrB  Zfixed_input_sizer2  r
  Z
first_convÚ
classifier)r   r   )r…  ÚkwargsrO   rO   rP   Ú_cfg+  s    õôrŠ  ztimm/)Ú	hf_hub_idz•https://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npz)r…  r‹  Úcustom_loadz•https://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz)r<   é€  r�  ç      ð?)r…  r‹  rŒ  r†  r‡  z–https://storage.googleapis.com/vit_models/augreg/S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzz–https://storage.googleapis.com/vit_models/augreg/S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npzz–https://storage.googleapis.com/vit_models/augreg/S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzz–https://storage.googleapis.com/vit_models/augreg/S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npzz—https://storage.googleapis.com/vit_models/augreg/B_32-i21k-300ep-lr_0.001-aug_medium1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzz•https://storage.googleapis.com/vit_models/augreg/B_32-i21k-300ep-lr_0.001-aug_light1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npzz–https://storage.googleapis.com/vit_models/augreg/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npzz–https://storage.googleapis.com/vit_models/augreg/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_384.npzz•https://storage.googleapis.com/vit_models/augreg/B_8-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npzz–https://storage.googleapis.com/vit_models/augreg/L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz–https://storage.googleapis.com/vit_models/augreg/L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npzzohttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_base_p16_224-80ecf9dd.pth)r…  r‹  zohttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_base_p16_384-83fb41ba.pth)r…  r‹  r†  r‡  zphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_large_p32_384-9b920ba8.pthz•https://storage.googleapis.com/vit_models/augreg/S_16-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npzz•https://storage.googleapis.com/vit_models/augreg/S_16-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_384.npzz•https://storage.googleapis.com/vit_models/augreg/B_32-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz•https://storage.googleapis.com/vit_models/augreg/B_32-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npzz•https://storage.googleapis.com/vit_models/augreg/B_16-i1k-300ep-lr_0.001-aug_strong2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz•https://storage.googleapis.com/vit_models/augreg/B_16-i1k-300ep-lr_0.001-aug_strong2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npz)r…  )r‹  rÔ   zmhttps://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0.npziSU  )r…  r‹  rŒ  rÔ   znhttps://storage.googleapis.com/vit_models/augreg/S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0.npzznhttps://storage.googleapis.com/vit_models/augreg/S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0.npzzohttps://storage.googleapis.com/vit_models/augreg/B_32-i21k-300ep-lr_0.001-aug_medium1-wd_0.03-do_0.0-sd_0.0.npzznhttps://storage.googleapis.com/vit_models/augreg/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0.npzzmhttps://storage.googleapis.com/vit_models/augreg/B_8-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0.npzznhttps://storage.googleapis.com/vit_models/augreg/L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1.npzz:https://storage.googleapis.com/vit_models/sam/ViT-B_32.npz)r…  rŒ  r‹  z:https://storage.googleapis.com/vit_models/sam/ViT-B_16.npzz[https://dl.fbaipublicfiles.com/dino/dino_deitsmall16_pretrain/dino_deitsmall16_pretrain.pth)r…  r‹  r2  r
  rÔ   zYhttps://dl.fbaipublicfiles.com/dino/dino_deitsmall8_pretrain/dino_deitsmall8_pretrain.pthzWhttps://dl.fbaipublicfiles.com/dino/dino_vitbase16_pretrain/dino_vitbase16_pretrain.pthzUhttps://dl.fbaipublicfiles.com/dino/dino_vitbase8_pretrain/dino_vitbase8_pretrain.pthzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vits14/dinov2_vits14_pretrain.pthz
apache-2.0)r<   é  r�  )r…  r‹  Úlicenser2  r
  rÔ   r†  r‡  zNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitb14/dinov2_vitb14_pretrain.pthzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_pretrain.pthzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitg14/dinov2_vitg14_pretrain.pthzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vits14/dinov2_vits14_reg4_pretrain.pthzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitb14/dinov2_vitb14_reg4_pretrain.pthzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_reg4_pretrain.pthzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitg14/dinov2_vitg14_reg4_pretrain.pthz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/vit_base_patch16_224_in21k_miil-887286df.pth)r1   r1   r1   )rŽ  rŽ  rŽ  g      ì?rN  iÕ+  )r…  r‹  r2  r
  r‡  rB  rÔ   zhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/vit_base_patch16_224_1k_miil_84_4-2deb18e3.pth)r…  r‹  r2  r
  r‡  rB  z}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_base_patch16_rpn_224-sw-3b07e89d.pth)r<   éð   r‘  gffffffî?i-.  )r‹  r†  r‡  rÔ   )r<   é   r’  )r‹  r†  r‡  Zsquash)r‹  r†  r‡  Ú	crop_mode)r‹  r2  r
  )r‹  r2  r
  r‡  r†  )r<   éÀ  r”  )r‹  r2  r
  r‡  )r‹  r2  r
  r‡  r†  r“  )r<   éP  r•  )r2  r
  )r2  r
  rÔ   )r‹  r2  r
  rÔ   )r‹  r2  r
  r‡  rÔ   z%laion/CLIP-ViT-B-32-laion2B-s34B-b79Kzopen_clip_pytorch_model.biné   )r‹  Úhf_hub_filenamer2  r
  rÔ   z%laion/CLIP-ViT-B-16-laion2B-s34B-b88K)r‹  r—  r2  r
  r‡  rÔ   z%laion/CLIP-ViT-L-14-laion2B-s32B-b82KrÇ   z%laion/CLIP-ViT-H-14-laion2B-s32B-b79Ké   z%laion/CLIP-ViT-g-14-laion2B-s12B-b42Kz(laion/CLIP-ViT-bigG-14-laion2B-39B-b160ké   z)laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90Kz.laion/CLIP-ViT-B-32-256x256-DataComp-s34B-b86K)r‹  r—  r2  r
  r‡  r†  rÔ   z)laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90Kz)laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90Kzapple/DFN2B-CLIP-ViT-B-16zapple/DFN2B-CLIP-ViT-L-14)zDnatively QuickGELU, use quickgelu model variant for original results)r‹  r—  Únotesr2  r
  r‡  rÔ   zapple/DFN5B-CLIP-ViT-H-14zapple/DFN5B-CLIP-ViT-H-14-378)r<   éz  r›  )r‹  r—  r2  r
  rš  r‡  r†  rÔ   z facebook/metaclip-b32-fullcc2.5bzmetaclip_b32_fullcc2.5b.binzcc-by-nc-4.0)r‹  r—  r�  rš  r2  r
  r‡  rÔ   z facebook/metaclip-b16-fullcc2.5bzmetaclip_b16_fullcc2.5b.binz facebook/metaclip-l14-fullcc2.5bzmetaclip_l14_fullcc2.5b.binz facebook/metaclip-h14-fullcc2.5bzmetaclip_h14_fullcc2.5b.binz%timm/vit_base_patch32_clip_224.openai)r‹  rš  r2  r
  rÔ   z%timm/vit_base_patch16_clip_224.openaiz&timm/vit_large_patch14_clip_224.openai)r‹  rš  r2  r
  r‡  rÔ   z&timm/vit_large_patch14_clip_336.openai)r‹  r—  rš  r2  r
  r‡  r†  rÔ   )r…  r†  r‡  Zmit)r<   éÄ   rœ  )r‹  r�  r2  r
  r†  r‡  )r‹  r�  r2  r
  r†  r‡  r“  zEhttps://storage.googleapis.com/big_vision/flexivit/flexivit_s_i1k.npz)r…  rŒ  r‹  r†  r‡  zKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_s_i1k_600ep.npzzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_s_i1k_300ep.npzzEhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i1k.npzzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i1k_600ep.npzzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i1k_300ep.npzzMhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i21k_1000ep.npz)r…  rŒ  r‹  r†  r‡  rÔ   zLhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i21k_300ep.npzzEhttps://storage.googleapis.com/big_vision/flexivit/flexivit_l_i1k.npzzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_l_i1k_600ep.npzzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_l_i1k_300ep.npzzIhttps://storage.googleapis.com/big_vision/flexivit/vit_b16_i21k_300ep.npzzIhttps://storage.googleapis.com/big_vision/flexivit/vit_b30_i21k_300ep.npzzEhttps://dl.fbaipublicfiles.com/mae/pretrain/mae_pretrain_vit_base.pth)r…  r‹  r�  r2  r
  rÔ   zFhttps://dl.fbaipublicfiles.com/mae/pretrain/mae_pretrain_vit_large.pthzEhttps://dl.fbaipublicfiles.com/mae/pretrain/mae_pretrain_vit_huge.pthz?https://dl.fbaipublicfiles.com/ijepa/IN1K-vit.h.14-300e.pth.tar)r…  r�  r2  r
  rÔ   z@https://dl.fbaipublicfiles.com/ijepa/IN22K-vit.h.14-900e.pth.tarzEhttps://dl.fbaipublicfiles.com/ijepa/IN1K-vit.h.16-448px-300e.pth.tar)r…  r�  r†  r‡  r2  r
  rÔ   z@https://dl.fbaipublicfiles.com/ijepa/IN22K-vit.g.16-600e.pth.tarztimm/ViT-B-16-SigLIP)r‹  r—  rÔ   ztimm/ViT-B-16-SigLIP-256)r‹  r—  r†  rÔ   ztimm/ViT-B-16-SigLIP-384ztimm/ViT-B-16-SigLIP-512)r<   r–  r–  ztimm/ViT-L-16-SigLIP-256ztimm/ViT-L-16-SigLIP-384ztimm/ViT-SO400M-14-SigLIPztimm/ViT-SO400M-14-SigLIP-384)r†  )¢z*vit_base_patch16_224.augreg2_in21k_ft_in1kz*vit_base_patch16_384.augreg2_in21k_ft_in1kz)vit_base_patch8_224.augreg2_in21k_ft_in1kz)vit_tiny_patch16_224.augreg_in21k_ft_in1kz)vit_tiny_patch16_384.augreg_in21k_ft_in1kz*vit_small_patch32_224.augreg_in21k_ft_in1kz*vit_small_patch32_384.augreg_in21k_ft_in1kz*vit_small_patch16_224.augreg_in21k_ft_in1kz*vit_small_patch16_384.augreg_in21k_ft_in1kz)vit_base_patch32_224.augreg_in21k_ft_in1kz)vit_base_patch32_384.augreg_in21k_ft_in1kz)vit_base_patch16_224.augreg_in21k_ft_in1kz)vit_base_patch16_384.augreg_in21k_ft_in1kz(vit_base_patch8_224.augreg_in21k_ft_in1kz*vit_large_patch16_224.augreg_in21k_ft_in1kz*vit_large_patch16_384.augreg_in21k_ft_in1kz'vit_base_patch16_224.orig_in21k_ft_in1kz'vit_base_patch16_384.orig_in21k_ft_in1kz(vit_large_patch32_384.orig_in21k_ft_in1kz!vit_small_patch16_224.augreg_in1kz!vit_small_patch16_384.augreg_in1kz vit_base_patch32_224.augreg_in1kz vit_base_patch32_384.augreg_in1kz vit_base_patch16_224.augreg_in1kz vit_base_patch16_384.augreg_in1kzvit_large_patch14_224.untrainedzvit_huge_patch14_224.untrainedzvit_giant_patch14_224.untrainedz"vit_gigantic_patch14_224.untrainedzvit_base_patch32_224.orig_in21kzvit_base_patch16_224.orig_in21kú vit_large_patch32_224.orig_in21kz vit_large_patch16_224.orig_in21kúvit_huge_patch14_224.orig_in21kú!vit_tiny_patch16_224.augreg_in21kú"vit_small_patch32_224.augreg_in21kú"vit_small_patch16_224.augreg_in21kú!vit_base_patch32_224.augreg_in21kú!vit_base_patch16_224.augreg_in21kú vit_base_patch8_224.augreg_in21kú"vit_large_patch16_224.augreg_in21kzvit_base_patch32_224.sam_in1kzvit_base_patch16_224.sam_in1kúvit_small_patch16_224.dinoúvit_small_patch8_224.dinoúvit_base_patch16_224.dinoúvit_base_patch8_224.dinoz vit_small_patch14_dinov2.lvd142mzvit_base_patch14_dinov2.lvd142mz vit_large_patch14_dinov2.lvd142mz vit_giant_patch14_dinov2.lvd142mz%vit_small_patch14_reg4_dinov2.lvd142mz$vit_base_patch14_reg4_dinov2.lvd142mz%vit_large_patch14_reg4_dinov2.lvd142mz%vit_giant_patch14_reg4_dinov2.lvd142múvit_base_patch16_224_miil.in21kz'vit_base_patch16_224_miil.in21k_ft_in1kz vit_base_patch16_rpn_224.sw_in1kz#vit_medium_patch16_gap_240.sw_in12kz+vit_medium_patch16_gap_256.sw_in12k_ft_in1kz+vit_medium_patch16_gap_384.sw_in12k_ft_in1kÚvit_base_patch16_gap_224z/vit_base_patch32_clip_224.laion2b_ft_in12k_in1kz/vit_base_patch32_clip_384.laion2b_ft_in12k_in1kz/vit_base_patch32_clip_448.laion2b_ft_in12k_in1kz/vit_base_patch16_clip_224.laion2b_ft_in12k_in1kz/vit_base_patch16_clip_384.laion2b_ft_in12k_in1kz0vit_large_patch14_clip_224.laion2b_ft_in12k_in1kz0vit_large_patch14_clip_336.laion2b_ft_in12k_in1kz/vit_huge_patch14_clip_224.laion2b_ft_in12k_in1kz/vit_huge_patch14_clip_336.laion2b_ft_in12k_in1kz.vit_base_patch32_clip_224.openai_ft_in12k_in1kz.vit_base_patch32_clip_384.openai_ft_in12k_in1kz.vit_base_patch16_clip_224.openai_ft_in12k_in1kz.vit_base_patch16_clip_384.openai_ft_in12k_in1kz/vit_large_patch14_clip_224.openai_ft_in12k_in1kz/vit_large_patch14_clip_336.openai_ft_in12k_in1kz)vit_base_patch32_clip_224.laion2b_ft_in1kz)vit_base_patch16_clip_224.laion2b_ft_in1kz)vit_base_patch16_clip_384.laion2b_ft_in1kz*vit_large_patch14_clip_224.laion2b_ft_in1kz*vit_large_patch14_clip_336.laion2b_ft_in1kz)vit_huge_patch14_clip_224.laion2b_ft_in1kz)vit_huge_patch14_clip_336.laion2b_ft_in1kz(vit_base_patch32_clip_224.openai_ft_in1kz(vit_base_patch16_clip_224.openai_ft_in1kz(vit_base_patch16_clip_384.openai_ft_in1kz)vit_large_patch14_clip_224.openai_ft_in1kz*vit_base_patch32_clip_224.laion2b_ft_in12kz*vit_base_patch16_clip_224.laion2b_ft_in12kz+vit_large_patch14_clip_224.laion2b_ft_in12kz*vit_huge_patch14_clip_224.laion2b_ft_in12kz)vit_base_patch32_clip_224.openai_ft_in12kz)vit_base_patch16_clip_224.openai_ft_in12kz*vit_large_patch14_clip_224.openai_ft_in12kú!vit_base_patch32_clip_224.laion2bz!vit_base_patch16_clip_224.laion2bú"vit_large_patch14_clip_224.laion2bú!vit_huge_patch14_clip_224.laion2bú"vit_giant_patch14_clip_224.laion2bz%vit_gigantic_patch14_clip_224.laion2bz$vit_base_patch32_clip_224.datacompxlz$vit_base_patch32_clip_256.datacompxlz$vit_base_patch16_clip_224.datacompxlz%vit_large_patch14_clip_224.datacompxlzvit_base_patch16_clip_224.dfn2bú vit_large_patch14_clip_224.dfn2búvit_huge_patch14_clip_224.dfn5búvit_huge_patch14_clip_378.dfn5bú(vit_base_patch32_clip_224.metaclip_2pt5bú(vit_base_patch16_clip_224.metaclip_2pt5bú)vit_large_patch14_clip_224.metaclip_2pt5bú(vit_huge_patch14_clip_224.metaclip_2pt5bú vit_base_patch32_clip_224.openaiú vit_base_patch16_clip_224.openaiú!vit_large_patch14_clip_224.openaiú!vit_large_patch14_clip_336.openaiz#vit_base_patch32_plus_256.untrainedz#vit_base_patch16_plus_240.untrainedz$vit_small_patch16_36x1_224.untrainedz$vit_small_patch16_18x2_224.untrainedz#vit_base_patch16_18x2_224.untrainedz)eva_large_patch14_196.in22k_ft_in22k_in1kz)eva_large_patch14_336.in22k_ft_in22k_in1kz#eva_large_patch14_196.in22k_ft_in1kz#eva_large_patch14_336.in22k_ft_in1kzflexivit_small.1200ep_in1kzflexivit_small.600ep_in1kzflexivit_small.300ep_in1kzflexivit_base.1200ep_in1kzflexivit_base.600ep_in1kzflexivit_base.300ep_in1kzflexivit_base.1000ep_in21kzflexivit_base.300ep_in21kzflexivit_large.1200ep_in1kzflexivit_large.600ep_in1kzflexivit_large.300ep_in1kzflexivit_base.patch16_in21kzflexivit_base.patch30_in21kz!vit_base_patch16_xp_224.untrainedz"vit_large_patch14_xp_224.untrainedz!vit_huge_patch14_xp_224.untrainedzvit_base_patch16_224.maezvit_large_patch16_224.maezvit_huge_patch14_224.maez#vit_huge_patch14_gap_224.in1k_ijepaz$vit_huge_patch14_gap_224.in22k_ijepaz#vit_huge_patch16_gap_448.in1k_ijepaz%vit_giant_patch16_gap_224.in22k_ijepaz!vit_base_patch16_siglip_224.webliz!vit_base_patch16_siglip_256.webliz!vit_base_patch16_siglip_384.webliz!vit_base_patch16_siglip_512.webliz"vit_large_patch16_siglip_256.webliz"vit_large_patch16_siglip_384.webliz#vit_so400m_patch14_siglip_224.webliz#vit_so400m_patch14_siglip_384.webliÚvit_medium_patch16_reg4_256Úvit_medium_patch16_reg4_gap_256Úvit_base_patch16_reg4_gap_256Úvit_so150m_patch16_reg4_gap_256Úvit_so150m_patch16_reg4_map_256r°  r±  r²  r³  r´  rµ  r¶  r·  r¸  r¹  rº  c                 C   s   i | ]}|  d d¡t| “qS )Z_clip_Z_clip_quickgelu_)rs  Údefault_cfgsr^  rO   rO   rP   r  ß  s    
 r  )ÚvariantÚ
pretrainedr9   c                 K   sj   |  dd ¡rtdƒ‚d| kr,ttddd�}nt}d}d| krP|  d	d ¡d
krPd}tt| |f||dœ|—ŽS )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.ZflexirN  F)rB  rC  TZsigliprÕ   rÌ   )Zpretrained_filter_fnZpretrained_strict)r€  ÚRuntimeErrorr   r„  r&   r-   )rÁ  rÂ  r‰  Z
_filter_fnÚstrictrO   rO   rP   Ú_create_vision_transformerå  s$    ýûúrÅ  )rÂ  r9   c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )	z ViT-Tiny (Vit-Ti/16)
    rÄ   éÀ   rÈ   r<   ©rÒ   rÖ   r×   r3   Úvit_tiny_patch16_224rÂ  )rÈ  ©rò   rÅ  ©rÂ  r‰  Ú
model_argsrH  rO   rO   rP   rÈ  ÿ  s    rÈ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )	z% ViT-Tiny (Vit-Ti/16) @ 384x384.
    rÄ   rÆ  rÈ   r<   rÇ  Úvit_tiny_patch16_384rÂ  )rÌ  rÉ  rÊ  rO   rO   rP   rÌ    s    rÌ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )	z ViT-Small (ViT-S/32)
    é    r�  rÈ   é   rÇ  Úvit_small_patch32_224rÂ  )rÏ  rÉ  rÊ  rO   rO   rP   rÏ    s    rÏ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )	z& ViT-Small (ViT-S/32) at 384x384.
    rÍ  r�  rÈ   rÎ  rÇ  Úvit_small_patch32_384rÂ  )rÐ  rÉ  rÊ  rO   rO   rP   rÐ    s    rÐ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )	ú ViT-Small (ViT-S/16)
    rÄ   r�  rÈ   rÎ  rÇ  Úvit_small_patch16_224rÂ  )rÒ  rÉ  rÊ  rO   rO   rP   rÒ  #  s    rÒ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )	rÑ  rÄ   r�  rÈ   rÎ  rÇ  Úvit_small_patch16_384rÂ  )rÓ  rÉ  rÊ  rO   rO   rP   rÓ  ,  s    rÓ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )	z ViT-Small (ViT-S/8)
    r0   r�  rÈ   rÎ  rÇ  Úvit_small_patch8_224rÂ  )rÔ  rÉ  rÊ  rO   rO   rP   rÔ  5  s    rÔ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )z¾ ViT-Base (ViT-B/32) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k, source https://github.com/google-research/vision_transformer.
    rÍ  rÇ   rÈ   rÇ  Úvit_base_patch32_224rÂ  )rÕ  rÉ  rÊ  rO   rO   rP   rÕ  >  s    rÕ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )zÎ ViT-Base model (ViT-B/32) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    rÍ  rÇ   rÈ   rÇ  Úvit_base_patch32_384rÂ  )rÖ  rÉ  rÊ  rO   rO   rP   rÖ  H  s    rÖ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )zÈ ViT-Base (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer.
    rÄ   rÇ   rÈ   rÇ  Úvit_base_patch16_224rÂ  )r×  rÉ  rÊ  rO   rO   rP   r×  R  s    r×  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )zÎ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    rÄ   rÇ   rÈ   rÇ  Úvit_base_patch16_384rÂ  )rØ  rÉ  rÊ  rO   rO   rP   rØ  \  s    rØ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )zÇ ViT-Base (ViT-B/8) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer.
    r0   rÇ   rÈ   rÇ  Úvit_base_patch8_224rÂ  )rÙ  rÉ  rÊ  rO   rO   rP   rÙ  f  s    rÙ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )	zo ViT-Large model (ViT-L/32) from original paper (https://arxiv.org/abs/2010.11929). No pretrained weights.
    rÍ  r˜  é   rÄ   rÇ  Úvit_large_patch32_224rÂ  )rÛ  rÉ  rÊ  rO   rO   rP   rÛ  p  s    rÛ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )	zÏ ViT-Large model (ViT-L/32) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    rÍ  r˜  rÚ  rÄ   rÇ  Úvit_large_patch32_384rÂ  )rÜ  rÉ  rÊ  rO   rO   rP   rÜ  y  s    rÜ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )zÏ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer.
    rÄ   r˜  rÚ  rÇ  Úvit_large_patch16_224rÂ  )rÝ  rÉ  rÊ  rO   rO   rP   rÝ  ƒ  s    rÝ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )zÏ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    rÄ   r˜  rÚ  rÇ  Úvit_large_patch16_384rÂ  )rÞ  rÉ  rÊ  rO   rO   rP   rÞ  �  s    rÞ  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )	z  ViT-Large model (ViT-L/14)
    é   r˜  rÚ  rÄ   rÇ  Úvit_large_patch14_224rÂ  )rà  rÉ  rÊ  rO   rO   rP   rà  —  s    rà  c                 K   s.   t ddddd�}tdd| it |f|Ž—Ž}|S )	zW ViT-Huge model (ViT-H/14) from original paper (https://arxiv.org/abs/2010.11929).
    rß  r™  rÍ  rÄ   rÇ  Úvit_huge_patch14_224rÂ  )rá  rÉ  rÊ  rO   rO   rP   rá     s    rá  c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
zq ViT-Giant (little-g) model (ViT-g/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    rß  é€  çtÑE]t@é(   rÄ   ©rÒ   rÖ   r�   r×   r3   Úvit_giant_patch14_224rÂ  )ræ  rÉ  rÊ  rO   rO   rP   ræ  ©  s    ræ  c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
zq ViT-Gigantic (big-G) model (ViT-G/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    rß  é€  ç;±;±@é0   rÄ   rå  Úvit_gigantic_patch14_224rÂ  )rê  rÉ  rÊ  rO   rO   rP   rê  ²  s     ÿÿ
ÿrê  c                 K   s0   t dddddd�}tdd| it |f|Ž—Ž}|S )	z• ViT-Base (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    Weights taken from: https://github.com/Alibaba-MIIL/ImageNet21K
    rÄ   rÇ   rÈ   F)rÒ   rÖ   r×   r3   r4   Úvit_base_patch16_224_miilrÂ  )rë  rÉ  rÊ  rO   rO   rP   rë  ¼  s     ÿÿ
ÿrë  c                 K   s8   t dddddddddd�	}tdd
| it |f|Ž—Ž}|S )zB ViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 240x240
    rÄ   r–  rÈ   r0   FrË   rè   ©	rÒ   rÖ   r×   r3   rØ   rÕ   r4   rx   rÜ   Úvit_medium_patch16_gap_240rÂ  )rí  rÉ  rÊ  rO   rO   rP   rí  Ç  s&           þ ÿÿ
ÿrí  c                 K   s8   t dddddddddd�	}tdd
| it |f|Ž—Ž}|S )zB ViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 256x256
    rÄ   r–  rÈ   r0   FrË   rè   rì  Úvit_medium_patch16_gap_256rÂ  )rî  rÉ  rÊ  rO   rO   rP   rî  Ó  s&           þ ÿÿ
ÿrî  c                 K   s8   t dddddddddd�	}tdd
| it |f|Ž—Ž}|S )zB ViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 384x384
    rÄ   r–  rÈ   r0   FrË   rè   rì  Úvit_medium_patch16_gap_384rÂ  )rï  rÉ  rÊ  rO   rO   rP   rï  ß  s&           þ ÿÿ
ÿrï  c              	   K   s4   t dddddddd�}td	d| it |f|Ž—Ž}|S )
z@ ViT-Base (ViT-B/16) w/o class token, w/ avg-pool @ 224x224
    rÄ   rÇ   rÈ   FrË   ©rÒ   rÖ   r×   r3   rØ   rÕ   rÜ   r«  rÂ  )r«  rÉ  rÊ  rO   rO   rP   r«  ë  s"          ÿ ÿÿ
ÿr«  c              	   K   s4   t dddddddd�}td
d	| it |f|Ž—Ž}|S )z; ViT-Huge model (ViT-H/14) w/ no class token, avg pool
    rß  r™  rÍ  rÄ   FrË   rð  Úvit_huge_patch14_gap_224rÂ  )rñ  rÉ  rÊ  rO   rO   rP   rñ  ö  s"          ÿ ÿÿ
ÿrñ  c              	   K   s4   t dddddddd�}td	d| it |f|Ž—Ž}|S )
zE ViT-Huge model (ViT-H/16) w/ no class token, avg pool @ 448x448
    rÄ   r™  rÍ  FrË   rð  Úvit_huge_patch16_gap_448rÂ  )rò  rÉ  rÊ  rO   rO   rP   rò    s"          ÿ ÿÿ
ÿrò  c              
   K   s6   t ddddddddd�}td
d	| it |f|Ž—Ž}|S )zH ViT-Giant (little-gg) model (ViT-g/16) w/ no class token, avg pool
    rÄ   râ  rä  rã  FrË   )rÒ   rÖ   r×   r3   r�   rØ   rÕ   rÜ   Úvit_giant_patch16_gap_224rÂ  )ró  rÉ  rÊ  rO   rO   rP   ró    s$          þ ÿÿ
ÿró  c                 K   s4   t dddddtjd�}tdd| it |f|Ž—Ž}|S )	ú) ViT-B/32 CLIP image tower @ 224x224
    rÍ  rÇ   rÈ   T©rÒ   rÖ   r×   r3   rÛ   r8   Úvit_base_patch32_clip_224rÂ  )rö  ©rò   rD   ro   rÅ  rÊ  rO   rO   rP   rö    s          ÿ ÿÿ
ÿrö  c                 K   s4   t dddddtjd�}tdd| it |f|Ž—Ž}|S )	z) ViT-B/32 CLIP image tower @ 256x256
    rÍ  rÇ   rÈ   Trõ  Úvit_base_patch32_clip_256rÂ  )rø  r÷  rÊ  rO   rO   rP   rø  #  s          ÿ ÿÿ
ÿrø  c                 K   s4   t dddddtjd�}tdd| it |f|Ž—Ž}|S )	z) ViT-B/32 CLIP image tower @ 384x384
    rÍ  rÇ   rÈ   Trõ  Úvit_base_patch32_clip_384rÂ  )rù  r÷  rÊ  rO   rO   rP   rù  .  s          ÿ ÿÿ
ÿrù  c                 K   s4   t dddddtjd�}tdd| it |f|Ž—Ž}|S )	z) ViT-B/32 CLIP image tower @ 448x448
    rÍ  rÇ   rÈ   Trõ  Úvit_base_patch32_clip_448rÂ  )rú  r÷  rÊ  rO   rO   rP   rú  9  s          ÿ ÿÿ
ÿrú  c                 K   s4   t dddddtjd�}tdd| it |f|Ž—Ž}|S )	z ViT-B/16 CLIP image tower
    rÄ   rÇ   rÈ   Trõ  Úvit_base_patch16_clip_224rÂ  )rû  r÷  rÊ  rO   rO   rP   rû  D  s     ÿÿ
ÿrû  c                 K   s4   t dddddtjd�}tdd| it |f|Ž—Ž}|S )	z) ViT-B/16 CLIP image tower @ 384x384
    rÄ   rÇ   rÈ   Trõ  Úvit_base_patch16_clip_384rÂ  )rü  r÷  rÊ  rO   rO   rP   rü  N  s     ÿÿ
ÿrü  c                 K   s4   t dddddtjd�}td	d| it |f|Ž—Ž}|S )
z1 ViT-Large model (ViT-L/14) CLIP image tower
    rß  r˜  rÚ  rÄ   Trõ  Úvit_large_patch14_clip_224rÂ  )rý  r÷  rÊ  rO   rO   rP   rý  X  s     ÿÿ
ÿrý  c                 K   s4   t dddddtjd�}td	d| it |f|Ž—Ž}|S )
z; ViT-Large model (ViT-L/14) CLIP image tower @ 336x336
    rß  r˜  rÚ  rÄ   Trõ  Úvit_large_patch14_clip_336rÂ  )rþ  r÷  rÊ  rO   rO   rP   rþ  b  s     ÿÿ
ÿrþ  c                 K   s4   t dddddtjd�}td	d| it |f|Ž—Ž}|S )
z1 ViT-Huge model (ViT-H/14) CLIP image tower.
    rß  r™  rÍ  rÄ   Trõ  Úvit_huge_patch14_clip_224rÂ  )rÿ  r÷  rÊ  rO   rO   rP   rÿ  l  s     ÿÿ
ÿrÿ  c                 K   s4   t dddddtjd�}td	d| it |f|Ž—Ž}|S )
z: ViT-Huge model (ViT-H/14) CLIP image tower @ 336x336
    rß  r™  rÍ  rÄ   Trõ  Úvit_huge_patch14_clip_336rÂ  )r   r÷  rÊ  rO   rO   rP   r   v  s     ÿÿ
ÿr   c                 K   s4   t dddddtjd�}td	d| it |f|Ž—Ž}|S )
z: ViT-Huge model (ViT-H/14) CLIP image tower @ 378x378
    rß  r™  rÍ  rÄ   Trõ  Úvit_huge_patch14_clip_378rÂ  )r  r÷  rÊ  rO   rO   rP   r  €  s     ÿÿ
ÿr  c              	   K   s6   t ddddddtjd�}td
d	| it |f|Ž—Ž}|S )zŸ ViT-Giant (little-g) model (ViT-g/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    Pretrained weights from CLIP image tower.
    rß  râ  rã  rä  rÄ   T©rÒ   rÖ   r�   r×   r3   rÛ   r8   Úvit_giant_patch14_clip_224rÂ  )r  r÷  rÊ  rO   rO   rP   r  Š  s"          ÿ ÿÿ
ÿr  c              	   K   s6   t ddddddtjd�}td
d	| it |f|Ž—Ž}|S )z“ ViT-bigG model (ViT-G/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    Pretrained weights from CLIP image tower.
    rß  rç  rè  ré  rÄ   Tr  Úvit_gigantic_patch14_clip_224rÂ  )r  r÷  rÊ  rO   rO   rP   r  –  s"          ÿ ÿÿ
ÿr  c              	   K   s6   t dddddtjdd�}td	d| it |f|Ž—Ž}|S )
rô  rÍ  rÇ   rÈ   TÚ
quick_gelu©rÒ   rÖ   r×   r3   rÛ   r8   rƒ   Ú#vit_base_patch32_clip_quickgelu_224rÂ  )r  r÷  rÊ  rO   rO   rP   r  ¢  s"         þ ÿÿ
ÿr  c              	   K   s6   t dddddtjdd�}td	d| it |f|Ž—Ž}|S )
z0 ViT-B/16 CLIP image tower w/ QuickGELU act
    rÄ   rÇ   rÈ   Tr  r  Ú#vit_base_patch16_clip_quickgelu_224rÂ  )r  r÷  rÊ  rO   rO   rP   r  ®  s"         þ ÿÿ
ÿr  c              	   K   sB   ddl m} tdddddtjdd	�}tdd| it|f|Ž—Ž}|S )zB ViT-Large model (ViT-L/14) CLIP image tower w/ QuickGELU act
    r   )r"   rß  r˜  rÚ  rÄ   Tr  r  Ú$vit_large_patch14_clip_quickgelu_224rÂ  )r	  )Útimm.layersr"   rò   rD   ro   rÅ  )rÂ  r‰  r"   rË  rH  rO   rO   rP   r	  º  s$         þ ÿÿ
ÿr	  c              	   K   s6   t dddddtjdd�}td
d	| it |f|Ž—Ž}|S )zL ViT-Large model (ViT-L/14) CLIP image tower @ 336x336 w/ QuickGELU act
    rß  r˜  rÚ  rÄ   Tr  r  Ú$vit_large_patch14_clip_quickgelu_336rÂ  )r  r÷  rÊ  rO   rO   rP   r  Ç  s"         þ ÿÿ
ÿr  c              	   K   s6   t dddddtjdd�}td
d	| it |f|Ž—Ž}|S )zB ViT-Huge model (ViT-H/14) CLIP image tower w/ QuickGELU act.
    rß  r™  rÍ  rÄ   Tr  r  Ú#vit_huge_patch14_clip_quickgelu_224rÂ  )r  r÷  rÊ  rO   rO   rP   r  Ó  s"         þ ÿÿ
ÿr  c              	   K   s6   t dddddtjdd�}td
d	| it |f|Ž—Ž}|S )zK ViT-Huge model (ViT-H/14) CLIP image tower @ 378x378 w/ QuickGELU act
    rß  r™  rÍ  rÄ   Tr  r  Ú#vit_huge_patch14_clip_quickgelu_378rÂ  )r  r÷  rÊ  rO   rO   rP   r  ß  s"         þ ÿÿ
ÿr  c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
z ViT-Base (ViT-B/32+)
    rÍ  é€  rÈ   rß  rw   ©rÒ   rÖ   r×   r3   rx   Úvit_base_patch32_plus_256rÂ  )r  rÉ  rÊ  rO   rO   rP   r  í  s     ÿÿ
ÿr  c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
z ViT-Base (ViT-B/16+)
    rÄ   r  rÈ   rß  rw   r  Úvit_base_patch16_plus_240rÂ  )r  rÉ  rÊ  rO   rO   rP   r  ÷  s     ÿÿ
ÿr  c                 K   s8   t dddddddtdd�	}td
d	| it |f|Ž—Ž}|S )z/ ViT-Base (ViT-B/16) w/ residual post-norm
    rÄ   rÇ   rÈ   Frw   rË   )	rÒ   rÖ   r×   r3   r4   rx   rØ   rç   rÕ   Úvit_base_patch16_rpn_224rÂ  )r  )rò   r“   rÅ  rÊ  rO   rO   rP   r  	  s&           þ ÿÿ
ÿr  c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
a   ViT-Base w/ LayerScale + 36 x 1 (36 block serial) config. Experimental, may remove.
    Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    Paper focuses on 24x2 + 48x1 for 'Small' width but those are extremely slow.
    rÄ   r�  é$   rÎ  rw   r  Úvit_small_patch16_36x1_224rÂ  )r  rÉ  rÊ  rO   rO   rP   r  	  s     ÿÿ
ÿr  c                 K   s2   t dddddtd�}td	d| it |f|Ž—Ž}|S )
a   ViT-Small w/ LayerScale + 18 x 2 (36 block parallel) config. Experimental, may remove.
    Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    Paper focuses on 24x2 + 48x1 for 'Small' width but those are extremely slow.
    rÄ   r�  é   rÎ  rw   ©rÒ   rÖ   r×   r3   rx   rç   Úvit_small_patch16_18x2_224rÂ  )r  ©rò   r«   rÅ  rÊ  rO   rO   rP   r  	  s          ÿ ÿÿ
ÿr  c                 K   s2   t dddddtd�}td	d| it |f|Ž—Ž}|S )
zÉ ViT-Base w/ LayerScale + 18 x 2 (36 block parallel) config. Experimental, may remove.
    Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    rÄ   rÇ   r  rÈ   rw   r  Úvit_base_patch16_18x2_224rÂ  )r  r  rÊ  rO   rO   rP   r  &	  s          ÿ ÿÿ
ÿr  c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
zG EVA-large model https://arxiv.org/abs/2211.07636 /via MAE MIM pretrainrß  r˜  rÚ  rÄ   rË   ©rÒ   rÖ   r×   r3   rÕ   Úeva_large_patch14_196rÂ  )r  rÉ  rÊ  rO   rO   rP   r  2	  s     ÿÿ
ÿr  c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
zF EVA-large model https://arxiv.org/abs/2211.07636 via MAE MIM pretrainrß  r˜  rÚ  rÄ   rË   r  Úeva_large_patch14_336rÂ  )r  rÉ  rÊ  rO   rO   rP   r  ;	  s    r  c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
z FlexiViT-Small
    rÄ   r�  rÈ   rÎ  T©rÒ   rÖ   r×   r3   rÙ   Úflexivit_smallrÂ  )r  rÉ  rÊ  rO   rO   rP   r  C	  s    r  c                 K   s0   t dddddd�}tdd| it |f|Ž—Ž}|S )	z FlexiViT-Base
    rÄ   rÇ   rÈ   Tr  Úflexivit_baserÂ  )r  rÉ  rÊ  rO   rO   rP   r  L	  s    r  c                 K   s0   t dddddd�}tdd| it |f|Ž—Ž}|S )	z FlexiViT-Large
    rÄ   r˜  rÚ  Tr  Úflexivit_largerÂ  )r   rÉ  rÊ  rO   rO   rP   r   U	  s    r   c                 K   s:   t ddddddttddd�
}td	d| it |f|Ž—Ž}|S )
úH ViT-Large model (ViT-L/14) w/ parallel blocks and qk norm enabled.
    rÄ   rÇ   rÈ   TF©
rÒ   rÖ   r×   r3   rÛ   rÙ   r8   rç   r4   r5   Úvit_base_patch16_xp_224rÂ  )r#  ©rò   r   rš   rÅ  rÊ  rO   rO   rP   r#  ^	  s(            þ ÿÿ
ÿr#  c                 K   s:   t ddddddttddd�
}td
d	| it |f|Ž—Ž}|S )r!  rß  r˜  rÚ  rÄ   TFr"  Úvit_large_patch14_xp_224rÂ  )r%  r$  rÊ  rO   rO   rP   r%  k	  s(            þ ÿÿ
ÿr%  c                 K   s:   t ddddddttddd�
}td
d	| it |f|Ž—Ž}|S )zG ViT-Huge model (ViT-H/14) w/ parallel blocks and qk norm enabled.
    rß  r™  rÍ  rÄ   TFr"  Úvit_huge_patch14_xp_224rÂ  )r&  r$  rÊ  rO   rO   rP   r&  x	  s(            þ ÿÿ
ÿr&  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )z ViT-S/14 for DINOv2
    rß  r�  rÈ   rÎ  rw   r�  ©rÒ   rÖ   r×   r3   rx   rÑ   Úvit_small_patch14_dinov2rÂ  )r(  rÉ  rÊ  rO   rO   rP   r(  …	  s     ÿÿ
ÿr(  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
z ViT-B/14 for DINOv2
    rß  rÇ   rÈ   rw   r�  r'  Úvit_base_patch14_dinov2rÂ  )r)  rÉ  rÊ  rO   rO   rP   r)  �	  s     ÿÿ
ÿr)  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )z ViT-L/14 for DINOv2
    rß  r˜  rÚ  rÄ   rw   r�  r'  Úvit_large_patch14_dinov2rÂ  )r*  rÉ  rÊ  rO   rO   rP   r*  ™	  s     ÿÿ
ÿr*  c                 K   s:   t ddddddtdtjd�	}tdd
| it |f|Ž—Ž}|S )ú ViT-G/14 for DINOv2
    rß  é   rä  rÚ  rw   çhË¹WU@r�  )	rÒ   rÖ   r×   r3   rx   r�   r„   rÑ   rƒ   Úvit_giant_patch14_dinov2rÂ  )r.  ©rò   r   rD   ZSiLUrÅ  rÊ  rO   rO   rP   r.  £	  s&           þ ÿÿ
ÿr.  c              	   K   s4   t dddddddd�}tdd
| it |f|Ž—Ž}|S )z( ViT-S/14 for DINOv2 w/ 4 registers
    rß  r�  rÈ   rÎ  rw   rT   T©rÒ   rÖ   r×   r3   rx   rÚ   rÙ   Úvit_small_patch14_reg4_dinov2rÂ  )r1  rÉ  rÊ  rO   rO   rP   r1  ´	  s"         þ ÿÿ
ÿr1  c              	   K   s4   t dddddddd�}td
d	| it |f|Ž—Ž}|S )z( ViT-B/14 for DINOv2 w/ 4 registers
    rß  rÇ   rÈ   rw   rT   Tr0  Úvit_base_patch14_reg4_dinov2rÂ  )r2  rÉ  rÊ  rO   rO   rP   r2  Á	  s"         þ ÿÿ
ÿr2  c              	   K   s4   t dddddddd�}tdd
| it |f|Ž—Ž}|S )z( ViT-L/14 for DINOv2 w/ 4 registers
    rß  r˜  rÚ  rÄ   rw   rT   Tr0  Úvit_large_patch14_reg4_dinov2rÂ  )r3  rÉ  rÊ  rO   rO   rP   r3  Î	  s"         þ ÿÿ
ÿr3  c                 K   s<   t ddddddttjddd	�
}tdd| it |f|Ž—Ž}|S )r+  rß  r,  rä  rÚ  rw   r-  rT   T)
rÒ   rÖ   r×   r3   rx   r�   r„   rƒ   rÚ   rÙ   Úvit_giant_patch14_reg4_dinov2rÂ  )r4  r/  rÊ  rO   rO   rP   r4  Û	  s(            þ ÿÿ
ÿr4  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrÄ   rÇ   rÈ   FrÌ   ©rÒ   rÖ   r×   r3   rØ   rÕ   Úvit_base_patch16_siglip_224rÂ  )r6  rÉ  rÊ  rO   rO   rP   r6  ì	  s          ÿ ÿÿ
ÿr6  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrÄ   rÇ   rÈ   FrÌ   r5  Úvit_base_patch16_siglip_256rÂ  )r7  rÉ  rÊ  rO   rO   rP   r7  ö	  s          ÿ ÿÿ
ÿr7  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrÄ   rÇ   rÈ   FrÌ   r5  Úvit_base_patch16_siglip_384rÂ  )r8  rÉ  rÊ  rO   rO   rP   r8   
  s          ÿ ÿÿ
ÿr8  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrÄ   rÇ   rÈ   FrÌ   r5  Úvit_base_patch16_siglip_512rÂ  )r9  rÉ  rÊ  rO   rO   rP   r9  

  s          ÿ ÿÿ
ÿr9  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrÄ   r˜  rÚ  FrÌ   r5  Úvit_large_patch16_siglip_256rÂ  )r:  rÉ  rÊ  rO   rO   rP   r:  
  s          ÿ ÿÿ
ÿr:  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrÄ   r˜  rÚ  FrÌ   r5  Úvit_large_patch16_siglip_384rÂ  )r;  rÉ  rÊ  rO   rO   rP   r;  
  s          ÿ ÿÿ
ÿr;  c              	   K   s4   t dddddddd�}tdd
| it |f|Ž—Ž}|S )Nrß  é€  é   rÄ   çˆ…ZÓ¼ã@FrÌ   ©rÒ   rÖ   r×   r3   r�   rØ   rÕ   Úvit_so400m_patch14_siglip_224rÂ  )r@  rÉ  rÊ  rO   rO   rP   r@  (
  s"          ÿ ÿÿ
ÿr@  c              	   K   s4   t dddddddd�}tdd
| it |f|Ž—Ž}|S )Nrß  r<  r=  rÄ   r>  FrÌ   r?  Úvit_so400m_patch14_siglip_384rÂ  )rA  rÉ  rÊ  rO   rO   rP   rA  2
  s"          ÿ ÿÿ
ÿrA  c              	   K   s4   t dddddddd�}td
d	| it |f|Ž—Ž}|S )NrÄ   r–  rÈ   r0   TrT   )rÒ   rÖ   r×   r3   rØ   rÙ   rÚ   r»  rÂ  )r»  rÉ  rÊ  rO   rO   rP   r»  <
  s"         þ ÿÿ
ÿr»  c              
   K   s6   t ddddddddd	�}tdd| it |f|Ž—Ž}|S )NrÄ   r–  rÈ   r0   FTrT   rË   )rÒ   rÖ   r×   r3   rØ   rÙ   rÚ   rÕ   r¼  rÂ  )r¼  rÉ  rÊ  rO   rO   rP   r¼  G
  s$          þ ÿÿ
ÿr¼  c              
   K   s6   t ddddddddd�}tdd
| it |f|Ž—Ž}|S )NrÄ   rÇ   rÈ   FTrË   rT   )rÒ   rÖ   r×   r3   rØ   rÙ   rÕ   rÚ   r½  rÂ  )r½  rÉ  rÊ  rO   rO   rP   r½  R
  s$          þ ÿÿ
ÿr½  c              
   K   s6   t ddddddddd	�}tdd| it |f|Ž—Ž}|S )NrÄ   r  r  rß  çú~j¼t“@FrT   rÌ   )rÒ   rÖ   r×   r3   r�   rØ   rÚ   rÕ   r¿  rÂ  )r¿  rÉ  rÊ  rO   rO   rP   r¿  ]
  s$          þ ÿÿ
ÿr¿  c                 K   s8   t dddddddddd	�	}tdd| it |f|Ž—Ž}|S )NrÄ   r  r  rß  rB  FrT   rË   )	rÒ   rÖ   r×   r3   r�   rØ   rÚ   rÕ   rÜ   r¾  rÂ  )r¾  rÉ  rÊ  rO   rO   rP   r¾  h
  s&           þ ÿÿ
ÿr¾  rŸ  r   r¡  r¢  r£  r¤  r�  r¥  rž  zvit_base_patch32_224.samzvit_base_patch16_224.samr¦  r§  r¨  r©  rª  r¬  r­  r®  r¯  )Zvit_tiny_patch16_224_in21kZvit_small_patch32_224_in21kZvit_small_patch16_224_in21kZvit_base_patch32_224_in21kZvit_base_patch16_224_in21kZvit_base_patch8_224_in21kZvit_large_patch32_224_in21kZvit_large_patch16_224_in21kZvit_huge_patch14_224_in21kZvit_base_patch32_224_samZvit_base_patch16_224_samZvit_small_patch16_224_dinoZvit_small_patch8_224_dinoZvit_base_patch16_224_dinoZvit_base_patch8_224_dinoZvit_base_patch16_224_miil_in21kZ!vit_base_patch32_224_clip_laion2bZ"vit_large_patch14_224_clip_laion2bZ!vit_huge_patch14_224_clip_laion2bZ"vit_giant_patch14_224_clip_laion2b)rÉ   )rÉ   r1   )rÉ   )rÎ   r1   )r%   rO   r>  F)rÉ   )ro  )Fr>  T)rÉ   )F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)±rª   Úloggingr   Úcollectionsr   Ú	functoolsr   Útypingr   r   r   r   r   r	   r
   r   r   r   r   ÚImportErrorÚtyping_extensionsrs   Ztorch.nnrD   Ztorch.nn.functionalZ
functionalr\   Ztorch.utils.checkpointZ	torch.jitr   Z	timm.datar   r   r   r   r   r   r
  r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   Z_builderr&   Z_manipulater'   r(   r)   Ú	_registryr*   r+   r,   Ú__all__Ú	getLoggerrj   rE  rr   r.   rv   r~   r“   rš   r«   r-   r4  r  rq   r;  r=  r  rt   rp   rm   rG  Zno_gradr  rv  r{  r„  rŠ  rÀ  Z_quick_gelu_cfgsrñ   rÅ  rÈ  rÌ  rÏ  rÐ  rÒ  rÓ  rÔ  rÕ  rÖ  r×  rØ  rÙ  rÛ  rÜ  rÝ  rÞ  rà  rá  ræ  rê  rë  rí  rî  rï  r«  rñ  rò  ró  rö  rø  rù  rú  rû  rü  rý  rþ  rÿ  r   r  r  r  r  r  r	  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r#  r%  r&  r(  r)  r*  r.  r1  r2  r3  r4  r6  r7  r8  r9  r:  r;  r@  rA  r»  r¼  r½  r¿  r¾  rO   rO   rO   rP   Ú<module>   s*
  0 D
3.6_F  C
   ú
ù! ýü/ý  ûúGÿÿý  ýý  ýý  ýý  ýý  ýýý  ýþ ý ýý  ýý  ýý  ýýýýýþ ý ý ý ý ý ý ý þ þ  ý  ý  ý  ý   û   û   û   û   û   û   û   û    ý   ýþ  þ þ  þ þ   þ   þ  þ   ý  þ   ý  þ   ý þ   ý  þ   ý  þ   ý þ  þ   ý  þ   ý  þ   ý þ þ   ý  þ  þ  þ   þ   þ  þ  þ   þ  ý   ý   ý   ý   ý   ý   ý   ü   ý   ý   ý   ü   ü   û   û   û   û   û  ý  ý   ý    ü   ü    ü   ü    ü  ý  ý  ý  ý  ý  ý   ý   ý  ý  ý  ý   ý   ý  ü  ü  ü  ü  ü   û  üýüüüüüýüÿÿÿÿÿ€ € € € € í     õ
ÿ									







																				




ì