U
    ¯Ê¹jkŸ  ã                    @   sê  d Z ddlZddlmZmZmZmZ ddlZddlm	Z	 ddl
m	  mZ ddlmZ ddlmZmZmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z% dd	l&m'Z'm(Z( d
gZ)G dd„ de	j*ƒZ+G dd„ de	j*ƒZ,G dd„ de	j*ƒZ-G dd
„ d
e	j*ƒZ.dYdd„Z/dZdd„Z0d[dd„Z1e'e1dd�e1dddd d!�e1deeddd d"�e1deed#dd d"�e1dd$dd d!�e1dd$dd d!�e1dd$dd d!�e1dddd%�e1dddd%�e1dd$dd%�e1dd$dd%�e1dd$dd%�e1dd$dd d&d'�e1dd$dd d&d'�e1dd$dd d&d'�e1ddd(�e1ddd(�e1ddd(�e1ddd(�e1ddd(�e1d)d*d+d,�e1d-d*d+d,�e1d.d*d/d,�e1d0d*d1d,�e1d2d*ddd1d3�e1d4d*d+d,�e1d5d*d+d,�e1dd6�d7œƒZ2e(d\e.d8œd9d:„ƒZ3e(d]e.d8œd;d<„ƒZ4e(d^e.d8œd=d>„ƒZ5e(d_e.d8œd?d@„ƒZ6e(d`e.d8œdAdB„ƒZ7e(dae.d8œdCdD„ƒZ8e(dbe.d8œdEdF„ƒZ9e(dce.d8œdGdH„ƒZ:e(dde.d8œdIdJ„ƒZ;e(dee.d8œdKdL„ƒZ<e(dfe.d8œdMdN„ƒZ=e(dge.d8œdOdP„ƒZ>e(dhe.d8œdQdR„ƒZ?e(die.d8œdSdT„ƒZ@e(dje.d8œdUdV„ƒZAe(dke.d8œdWdX„ƒZBdS )la    EVA

EVA from https://github.com/baaivision/EVA , paper: https://arxiv.org/abs/2211.07636

@article{EVA,
  title={EVA: Exploring the Limits of Masked Visual Representation Learning at Scale},
  author={Fang, Yuxin and Wang, Wen and Xie, Binhui and Sun, Quan and Wu, Ledell and Wang, Xinggang and Huang,
  Tiejun and Wang, Xinlong and Cao, Yue},
  journal={arXiv preprint arXiv:2211.07636},
  year={2022}
}

EVA-02: A Visual Representation for Neon Genesis - https://arxiv.org/abs/2303.11331
@article{EVA02,
  title={EVA-02: A Visual Representation for Neon Genesis},
  author={Fang, Yuxin and Sun, Quan and Wang, Xinggang and Huang, Tiejun and Wang, Xinlong and Cao, Yue},
  journal={arXiv preprint arXiv:2303.11331},
  year={2023}
}

This file contains EVA & EVA02 model implementations evolved from BEiT, additional models in vision_transformer.py.

Modifications by / Copyright 2023 Ross Wightman, original copyrights below
é    N)ÚCallableÚOptionalÚTupleÚUnion)Ú
checkpoint)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STDÚOPENAI_CLIP_MEANÚOPENAI_CLIP_STD)Ú
PatchEmbedÚMlpÚGluMlpÚSwiGLUÚ	LayerNormÚDropPathÚPatchDropoutÚRotaryEmbeddingCatÚapply_rot_embed_catÚapply_keep_indices_nlcÚtrunc_normal_Úresample_patch_embedÚresample_abs_pos_embedÚ	to_2tupleÚuse_fused_attné   )Úbuild_model_with_cfg)Úgenerate_default_cfgsÚregister_modelÚEvac                
       sj   e Zd ZU ejje ed< deeeee	e	e
e e
e dœ‡ fdd„Zde
ej e
ej d	œd
d„Z‡  ZS )ÚEvaAttentionÚ
fused_attné   Tç        N)ÚdimÚ	num_headsÚqkv_biasÚ	qkv_fusedÚ	attn_dropÚ	proj_dropÚattn_head_dimÚ
norm_layerc	                    sT  t ƒ  ¡  || _|| }	|dk	r$|}	|	| j }
|	d | _tƒ | _|rÀtj||
d dd�| _d | _	 | _
| _|r¬t t |
¡¡| _| jdt |
¡dd� t t |
¡¡| _nd | _ | _| _nNtj||
|d�| _	tj||
dd�| _
tj||
|d�| _d| _d | _ | _| _t |¡| _|dk	�r,||
ƒnt ¡ | _t |
|¡| _t |¡| _dS )zÎ

        Args:
            dim:
            num_heads:
            qkv_bias:
            qkv_fused:
            attn_drop:
            proj_drop:
            attn_head_dim:
            norm_layer:
        Ng      à¿é   F)ÚbiasÚk_bias)Ú
persistent)ÚsuperÚ__init__r$   Úscaler   r    ÚnnÚLinearÚqkvÚq_projÚk_projÚv_projÚ	ParameterÚtorchÚzerosÚq_biasZregister_bufferÚv_biasr-   ÚDropoutr'   ÚIdentityÚnormÚprojr(   )Úselfr#   r$   r%   r&   r'   r(   r)   r*   Zhead_dimZall_head_dim©Ú	__class__© úN/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/eva.pyr0   2   s2    


zEvaAttention.__init__©ÚropeÚ	attn_maskc              
   C   sn  |j \}}}| jd k	r‚| jd k	r6t | j| j| jf¡nd }tj|| jj	|d�}| 
||d| jd¡ ddddd¡}| d¡\}	}
}n`|  |¡ 
||| jd¡ dd¡}	|  |¡ 
||| jd¡ dd¡}
|  |¡ 
||| jd¡ dd¡}|d k	�r”t |	d d …d d …d d…d d …f t|	d d …d d …dd …d d …f |ƒgd¡ |¡}	t |
d d …d d …d d…d d …f t|
d d …d d …dd …d d …f |ƒgd¡ |¡}
| j�rÂtj|	|
||| j�r¸| jjndd	�}nt|	| j }	|	|
 d
d¡ }|jdd�}|d k	�r$| tj¡}| |d d …d d d d …f  tdƒ¡}|  |¡}|| }| dd¡ 
|||¡}|  |¡}|   |¡}|  !|¡}|S )N)ÚinputÚweightr,   r+   éÿÿÿÿé   r   r   é   r"   )rH   Z	dropout_péþÿÿÿ©r#   z-inf)"Úshaper4   r;   r9   Úcatr-   r<   ÚFZlinearrJ   Zreshaper$   ZpermuteZunbindr5   Z	transposer6   r7   r   Ztype_asr    Zscaled_dot_product_attentionZtrainingr'   Úpr1   ZsoftmaxÚtoÚboolZmasked_fillÚfloatr?   r@   r(   )rA   ÚxrG   rH   ÚBÚNÚCr%   r4   ÚqÚkÚvÚattnrD   rD   rE   Úforwardg   sB    
$"   
TT  ý

&



zEvaAttention.forward)r!   TTr"   r"   NN)NN)Ú__name__Ú
__module__Ú__qualname__r9   ÚjitÚFinalrU   Ú__annotations__ÚintrV   r   r   r0   ÚTensorr_   Ú__classcell__rD   rD   rB   rE   r   /   s0   
       ÷÷8  üür   c                       s‚   e Zd Zddddddddddejedfeeeeeeeeeeee	e e
e
e	e dœ‡ fdd„Zde	ej e	ej d	œd
d„Z‡  ZS )ÚEvaBlockTç      @Fr"   N©r#   r$   r%   r&   Ú	mlp_ratioÚ
swiglu_mlpÚ	scale_mlpÚscale_attn_innerr(   r'   Ú	drop_pathÚinit_valuesÚ	act_layerr*   r)   c              
      s2  t ƒ  ¡  ||ƒ| _t|||||
|	||r,|ndd�| _|dk	rRt |t |¡ ¡nd| _	|dkrht
|ƒnt ¡ | _||ƒ| _t|| ƒ}|rÒ|r¬t|||rž|nd|	d�| _qît||d |r¾|ndtjd|	d�| _nt||||râ|nd|	d�| _|dk	�rt |t |¡ ¡nd| _|dk�r$t
|ƒnt ¡ | _dS ©	ay  

        Args:
            dim:
            num_heads:
            qkv_bias:
            qkv_fused:
            mlp_ratio:
            swiglu_mlp:
            scale_mlp:
            scale_attn_inner:
            proj_drop:
            attn_drop:
            drop_path:
            init_values:
            act_layer:
            norm_layer:
            attn_head_dim:
        N)r$   r%   r&   r'   r(   r)   r*   r"   )Úin_featuresÚhidden_featuresr*   ÚdroprL   F)rt   ru   r*   rr   Z	gate_lastrv   )rt   ru   rr   r*   rv   )r/   r0   Únorm1r   r^   r2   r8   r9   ZonesÚgamma_1r   r>   Ú
drop_path1Únorm2rf   r   Úmlpr   ÚSiLUr   Úgamma_2Ú
drop_path2©rA   r#   r$   r%   r&   rl   rm   rn   ro   r(   r'   rp   rq   rr   r*   r)   ru   rB   rD   rE   r0   –   sR    %


ø
"

ü

ú
	
û$zEvaBlock.__init__rF   c              	   C   s�   | j d krF||  | j|  |¡||d�¡ }||  |  |  |¡¡¡ }nF||  | j | j|  |¡||d� ¡ }||  | j|  |  |¡¡ ¡ }|S ©NrF   )rx   ry   r^   rw   r~   r{   rz   r}   ©rA   rW   rG   rH   rD   rD   rE   r_   ê   s    
 & zEvaBlock.forward)NN)r`   ra   rb   r2   ÚGELUr   rf   rU   rV   r   r   r0   r9   rg   r_   rh   rD   rD   rB   rE   ri   ”   s>   ððTri   c                       sˆ   e Zd ZdZddddddddddejejdfeeeee	eeee	e	e	e
e	 eee
e dœ‡ fdd	„Zde
ej e
ej d
œdd„Z‡  ZS )ÚEvaBlockPostNormzF EVA block w/ post-norm and support for swiglu, MLP norm scale, ROPE. Trj   Fr"   Nrk   c              
      sê   t ƒ  ¡  t|||||
|	||r"|ndd�| _||ƒ| _|dkrFt|ƒnt ¡ | _t	|| ƒ}|r¦|r€t
|||rr|nd|	d�| _qÂt||d |r’|ndtjd|	d�| _nt||||r¶|nd|	d�| _||ƒ| _|dkrÜt|ƒnt ¡ | _dS rs   )r/   r0   r   r^   rw   r   r2   r>   ry   rf   r   r{   r   r|   r   rz   r~   r   rB   rD   rE   r0   ö   sN    %

ø


ü

ú
	
û
zEvaBlockPostNorm.__init__rF   c              
   C   s>   ||   |  | j|||d�¡¡ }||  |  |  |¡¡¡ }|S r€   )ry   rw   r^   r~   rz   r{   r�   rD   rD   rE   r_   H  s     zEvaBlockPostNorm.forward)NN)r`   ra   rb   Ú__doc__r2   r‚   r   rf   rU   rV   r   r   r0   r9   rg   r_   rh   rD   rD   rB   rE   rƒ   ô   s@   ððRrƒ   c                "       sr  e Zd ZdZddddddddd	d	d
dddddddddedd	d	ddddddfeeeeef f eeeeef f eeeeeee	e	e
e	e	e	e
e
e
e
e
e
eee
 e	e	e	e	e	e	eeeeef ef  e
dœ‡ fdd„Zdd„ Zdd„ Zejjdd„ ƒZejjd*dd„ƒZejjd+dd„ƒZejjdd„ ƒZd,dd„Zeejeej f d œd!d"„Zd#d$„ Zd-e	d%œd&d'„Zd(d)„ Z‡  ZS ).r   a!   Eva Vision Transformer w/ Abs & Rotary Pos Embed

    This class implements the EVA and EVA02 models that were based on the BEiT ViT variant
      * EVA - abs pos embed, global avg pool
      * EVA02 - abs + rope pos embed, global avg pool, SwiGLU, scale Norm in MLP (ala normformer)
    éà   é   r+   éè  Úavgé   é   Trj   Fr"   Ngü©ñÒMbP?)Úimg_sizeÚ
patch_sizeÚin_chansÚnum_classesÚglobal_poolÚ	embed_dimÚdepthr$   r%   r&   rl   rm   rn   ro   Ú	drop_rateÚpos_drop_rateÚpatch_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rater*   rq   Úclass_tokenÚuse_abs_pos_embÚuse_rot_pos_embÚuse_post_normÚdynamic_img_sizeÚdynamic_img_padÚref_feat_shapeÚhead_init_scalec           "         s„  t ƒ  ¡  || _|| _ˆ | _| _|r*dnd| _|| _d| _i }|rV| 	t
ddd�¡ tf |||ˆ|dœ|—Ž| _| jj} |r’t t ddˆ¡¡nd| _|r¶t t d| | j ˆ¡¡nd| _tj|d�| _|dkræt|| jd	d
�| _nd| _|�r.|dk	�rt|ƒnd}tˆˆ d|�rdn| jj|d�| _nd| _dd„ t d||¡D ƒ‰|�rVtnt‰t ‡ ‡‡‡‡‡‡‡‡‡	‡
‡‡‡fdd„t |ƒD ƒ¡| _!| jdk}!|!�rªt "¡ nˆˆƒ| _#|!�rÂˆˆƒnt "¡ | _$t |¡| _%|dk�rît &ˆ|¡nt "¡ | _'|  (| j)¡ | jdk	�rt*| jdd� | jdk	�r8t*| jdd� |  +¡  t,| j'tj&ƒ�r€t*| j'j-dd� | j'j-j. /|¡ | j'j0j. /|¡ dS )a¾  

        Args:
            img_size:
            patch_size:
            in_chans:
            num_classes:
            global_pool:
            embed_dim:
            depth:
            num_heads:
            qkv_bias:
            qkv_fused:
            mlp_ratio:
            swiglu_mlp:
            scale_mlp:
            scale_attn_inner:
            drop_rate:
            pos_drop_rate:
            proj_drop_rate:
            attn_drop_rate:
            drop_path_rate:
            norm_layer:
            init_values:
            class_token:
            use_abs_pos_emb:
            use_rot_pos_emb:
            use_post_norm:
            ref_feat_shape:
            head_init_scale:
        r   r   FZNHWC)Zstrict_img_sizeZ
output_fmt)r‹   rŒ   r�   r�   r�   N)rS   T)Únum_prefix_tokensZreturn_indices)Z	in_pixelsZ
feat_shaperž   c                 S   s   g | ]}|  ¡ ‘qS rD   )Úitem)Ú.0rW   rD   rD   rE   Ú
<listcomp>Å  s     z Eva.__init__.<locals>.<listcomp>c                    s2   g | ]*}ˆˆˆˆ	ˆ
ˆˆˆˆˆˆ ˆ| ˆˆd �‘qS ))r#   r$   r%   r&   rl   rm   rn   ro   r(   r'   rp   r*   rq   rD   )r¢   Úi©r–   Zblock_fnZdprr�   rq   rl   r*   r$   r•   r%   r&   ro   rn   rm   rD   rE   r£   Ç  s    ñórˆ   ç{®Gáz”?©Ústd)1r/   r0   rŽ   r�   Znum_featuresr�   r    rœ   Úgrad_checkpointingÚupdateÚdictr   Úpatch_embedÚnum_patchesr2   r8   r9   r:   Ú	cls_tokenÚ	pos_embedr=   Úpos_dropr   Ú
patch_dropr   r   Ú	grid_sizerG   Zlinspacerƒ   ri   Z
ModuleListÚrangeÚblocksr>   r?   Úfc_normÚ	head_dropr3   ÚheadÚapplyÚ_init_weightsr   Úfix_init_weightÚ
isinstancerJ   ÚdataZmul_r,   )"rA   r‹   rŒ   r�   rŽ   r�   r�   r‘   r$   r%   r&   rl   rm   rn   ro   r’   r“   r”   r•   r–   r—   r*   rq   r˜   r™   rš   r›   rœ   r�   rž   rŸ   Z
embed_argsr­   Zuse_fc_normrB   r¥   rE   r0   V  s€    @
ûúÿÿÿý
ü
(ð

 zEva.__init__c                 C   sL   dd„ }t | jƒD ]4\}}||jjjj|d ƒ ||jjjj|d ƒ qd S )Nc                 S   s   |   t d| ¡¡ d S )Ng       @)Zdiv_ÚmathÚsqrt)ÚparamÚlayer_idrD   rD   rE   Úrescaleì  s    z$Eva.fix_init_weight.<locals>.rescaler   )Ú	enumerater´   r^   r@   rJ   r¼   r{   Zfc2)rA   rÁ   rÀ   ÚlayerrD   rD   rE   rº   ë  s    zEva.fix_init_weightc                 C   s6   t |tjƒr2t|jdd� |jd k	r2tj |j¡ d S )Nr¦   r§   )r»   r2   r3   r   rJ   r,   ÚinitZzeros_)rA   ÚmrD   rD   rE   r¹   ó  s    
zEva._init_weightsc                 C   s   ddh}|S )Nr¯   r®   rD   )rA   ZnwdrD   rD   rE   Úno_weight_decayù  s    zEva.no_weight_decayc                 C   s
   || _ d S ©N)r©   )rA   ÚenablerD   rD   rE   Úset_grad_checkpointingþ  s    zEva.set_grad_checkpointingc                 C   s   t dddgd�}|S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )Ústemr´   )r«   )rA   ZcoarseZmatcherrD   rD   rE   Úgroup_matcher  s
    þzEva.group_matcherc                 C   s   | j S rÇ   )r·   )rA   rD   rD   rE   Úget_classifier
  s    zEva.get_classifierc                 C   s8   || _ |d k	r|| _|dkr*t | j|¡nt ¡ | _d S )Nr   )rŽ   r�   r2   r3   r�   r>   r·   )rA   rŽ   r�   rD   rD   rE   Úreset_classifier  s    zEva.reset_classifier©Úreturnc           	      C   s  | j rj|j\}}}}| jd k	r6t| j||f| jd�}nd }| |d|¡}| jd k	rd| jj||fd�nd }n| j}| jd k	r„| j ¡ nd }| jd k	r¶t	j
| j |jd dd¡|fdd�}|d k	rÆ|| }|  |¡}| jd k	�r
|  |¡\}}|d k	�r
|d k	�r
t|||ƒ}||fS )N)r    rK   )rP   r   r   rO   )rœ   rP   r¯   r   r    ÚviewrG   Z	get_embedr®   r9   rQ   Úexpandr°   r±   r   )	rA   rW   rX   ÚHÚWrZ   r¯   Úrot_pos_embedZkeep_indicesrD   rD   rE   Ú
_pos_embed  s.    
ý"
$
zEva._pos_embedc                 C   s^   |   |¡}|  |¡\}}| jD ]0}| jrBtj ¡ sBt|||d�}q|||d�}q|  |¡}|S )N)rG   )	r¬   rÕ   r´   r©   r9   rc   Zis_scriptingr   r?   )rA   rW   rÔ   ZblkrD   rD   rE   Úforward_features2  s    


zEva.forward_features)Ú
pre_logitsc                 C   sd   | j r>| j dkr.|d d …| jd …f jdd�n|d d …df }|  |¡}|  |¡}|rZ|S |  |¡S )Nrˆ   r   rO   r   )r�   r    Úmeanrµ   r¶   r·   )rA   rW   r×   rD   rD   rE   Úforward_head=  s
    8

zEva.forward_headc                 C   s   |   |¡}|  |¡}|S rÇ   )rÖ   rÙ   )rA   rW   rD   rD   rE   r_   D  s    

zEva.forward)T)F)N)F)r`   ra   rb   r„   r   r   rf   r   ÚstrrU   rV   r   r   r0   rº   r¹   r9   rc   ÚignorerÆ   rÉ   rË   rÌ   rÍ   rg   rÕ   rÖ   rÙ   r_   rh   rD   rD   rB   rE   r   N  sž   	áá 


ÚbicubicTc              	   C   sü  i }|   d| ¡} |   d| ¡} |   d| ¡} |   d| ¡} d| krBd}nd| krPd}nd	}|d
 | k}|d | k}t|ƒ}|  ¡ D �]x\}	}
|r¤|	 |¡r||	|d… }	nq|d|	kr®q|d|	krþ|jjjj\}}}}|
jd |ksæ|
jd |krüt|
||f||dd�}
nX|	dk�rV|
jd |j	jd k�rVt
|ddƒ�r2dn
t
|ddƒ}t|
|jj|||dd�}
|	 dd¡}	|	 dd¡}	|	 dd¡}	|	 d d!¡}	|	 d"d#¡}	|	 d$d%¡}	|�r¼|	 d&d'¡}	|	 d(d)¡}	|�rî|	d*k�rî|	d+k�sÞ|	d,kr||	 d-d.¡}	nq||
||	< q||S )/zJ convert patch embedding weight from manual patchify + linear proj to convZ	model_emaÚmodelÚmoduleÚ
state_dictzvisual.trunk.pos_embedzvisual.trunk.zvisual.pos_embedzvisual.Ú Ú
mask_tokenzblocks.0.attn.q_proj.weightNrG   zpatch_embed.proj.weightrK   rN   T)ÚinterpolationÚ	antialiasÚverboser¯   r   Zno_embed_classFr   r    )Únew_sizer    râ   rã   rä   z
mlp.ffn_lnzmlp.normzattn.inner_attn_lnz	attn.normzmlp.w12zmlp.fc1zmlp.w1z	mlp.fc1_gzmlp.w2z	mlp.fc1_xzmlp.w3zmlp.fc2r;   zq_proj.biasr<   zv_proj.bias)rá   zlm_head.weightzlm_head.biasúnorm.weightú	norm.biasræ   rç   r?   rµ   )ÚgetÚlenÚitemsÚ
startswithr¬   r@   rJ   rP   r   r¯   Úgetattrr   r²   Úreplace)rß   rÝ   râ   rã   Zout_dictÚprefixZmim_weightsZno_qkvZ
len_prefixr\   r]   Ú_rÒ   rÓ   r    rD   rD   rE   Úcheckpoint_filter_fnJ  sn    
û"ú	
rð   Fc                 K   s0   |  dd ¡rtdƒ‚tt| |fdti|—Ž}|S )NZfeatures_onlyz-features_only not implemented for Eva models.Zpretrained_filter_fn)rè   ÚRuntimeErrorr   r   rð   )ÚvariantÚ
pretrainedÚkwargsrÝ   rD   rD   rE   Ú_create_eva™  s      ÿþýrõ   rà   c                 K   s"   | ddd dddt tdddd	œ|–S )
Nr‡   )r+   r…   r…   gÍÌÌÌÌÌì?rÜ   Tzpatch_embed.projr·   Zmit)ÚurlrŽ   Ú
input_sizeZ	pool_sizeÚcrop_pctrâ   Zfixed_input_sizerØ   r¨   Z
first_convÚ
classifierÚlicense)r	   r
   )rö   rô   rD   rD   rE   Ú_cfg¤  s          úúrû   ztimm/)Ú	hf_hub_id)r+   éP  rý   g      ð?Zsquash)rü   r÷   rø   Ú	crop_mode)rü   rØ   r¨   r÷   rø   rþ   )r+   é0  rÿ   )r+   éÀ  r   )rü   r÷   rø   iQU  )rü   r÷   rø   rþ   rŽ   )rü   rŽ   z3timm/eva_giant_patch14_clip_224.laion400m_s11b_b41kzopen_clip_pytorch_model.biné   )rü   Úhf_hub_filenamerŽ   z8timm/eva_giant_patch14_plus_clip_224.merged2b_s11b_b114kz3timm/eva02_base_patch16_clip_224.merged2b_s8b_b131ki   z4timm/eva02_large_patch14_clip_224.merged2b_s4b_b131kr‰   z3timm/eva02_large_patch14_clip_336.merged2b_s6b_b61k)rü   r  r÷   rø   rŽ   z6timm/eva02_enormous_patch14_clip_224.laion2b_s4b_b115kz;timm/eva02_enormous_patch14_plus_clip_224.laion2b_s9b_b144k)rŽ   )z"eva_giant_patch14_224.clip_ft_in1kz"eva_giant_patch14_336.clip_ft_in1kz(eva_giant_patch14_336.m30m_ft_in22k_in1kz(eva_giant_patch14_560.m30m_ft_in22k_in1kz.eva02_base_patch14_448.mim_in22k_ft_in22k_in1kz/eva02_large_patch14_448.mim_in22k_ft_in22k_in1kz.eva02_large_patch14_448.mim_m38m_ft_in22k_in1kz(eva02_tiny_patch14_336.mim_in22k_ft_in1kz)eva02_small_patch14_336.mim_in22k_ft_in1kz(eva02_base_patch14_448.mim_in22k_ft_in1kz)eva02_large_patch14_448.mim_in22k_ft_in1kz(eva02_large_patch14_448.mim_m38m_ft_in1kz)eva02_base_patch14_448.mim_in22k_ft_in22kz*eva02_large_patch14_448.mim_in22k_ft_in22kz)eva02_large_patch14_448.mim_m38m_ft_in22kz eva02_tiny_patch14_224.mim_in22kz!eva02_small_patch14_224.mim_in22kz eva02_base_patch14_224.mim_in22kz!eva02_large_patch14_224.mim_in22kz eva02_large_patch14_224.mim_m38mz$eva_giant_patch14_clip_224.laion400mz#eva_giant_patch14_clip_224.merged2bz$eva02_base_patch16_clip_224.merged2bz%eva02_large_patch14_clip_224.merged2bz%eva02_large_patch14_clip_336.merged2bz'eva02_enormous_patch14_clip_224.laion2bz,eva02_enormous_patch14_clip_224.laion2b_plusz(eva02_enormous_patch14_clip_224.pretrainrÎ   c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
ú. EVA-g model https://arxiv.org/abs/2211.07636 é   é€  é(   r†   çtÑE]t@©rŒ   r�   r‘   r$   rl   Úeva_giant_patch14_224ró   )r	  ©r«   rõ   ©ró   rô   Z
model_argsrÝ   rD   rD   rE   r	  S  s    r	  c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
r  r  r  r  r†   r  r  Úeva_giant_patch14_336ró   )r  r
  r  rD   rD   rE   r  [  s    r  c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
r  r  r  r  r†   r  r  Úeva_giant_patch14_560ró   )r  r
  r  rD   rD   rE   r  c  s    r  c                 K   s8   t dddddddddd	�	}tdd| it |f|Ž—Ž}|S )Nr…   r  éÀ   rŠ   r+   çUUUUUU@T©r†   r†   ©	r‹   rŒ   r�   r‘   r$   rl   rm   rš   rž   Úeva02_tiny_patch14_224ró   )r  r
  r  rD   rD   rE   r  k  s    ÷r  c                 K   s8   t dddddddddd	�	}tdd| it |f|Ž—Ž}|S )Nr…   r  é€  rŠ   é   r  Tr  r  Úeva02_small_patch14_224ró   )r  r
  r  rD   rD   rE   r  |  s    ÷r  c                 K   s<   t dddddddddddd	�}tdd| it |f|Ž—Ž}|S )Nr…   r  r‰   rŠ   Fr  Tr  ©r‹   rŒ   r�   r‘   r$   r&   rl   rm   rn   rš   rž   Úeva02_base_patch14_224ró   )r  r
  r  rD   rD   rE   r  �  s    õr  c                 K   s<   t ddddddddddd	d
�}tdd| it |f|Ž—Ž}|S )Nr…   r  r  é   r†   r  FTr  ©r‹   rŒ   r�   r‘   r$   rl   r&   rm   rn   rš   rž   Úeva02_large_patch14_224ró   )r  r
  r  rD   rD   rE   r     s    õr  c                 K   s8   t dddddddddd	�	}tdd| it |f|Ž—Ž}|S )Nrý   r  r  rŠ   r+   r  Tr  r  Úeva02_tiny_patch14_336ró   )r  r
  r  rD   rD   rE   r  ³  s    ÷r  c                 K   s8   t dddddddddd	�	}tdd| it |f|Ž—Ž}|S )Nrý   r  r  rŠ   r  r  Tr  r  Úeva02_small_patch14_336ró   )r  r
  r  rD   rD   rE   r  Ä  s    ÷r  c                 K   s<   t dddddddddddd	�}tdd| it |f|Ž—Ž}|S )Nr   r  r‰   rŠ   Fr  Tr  r  Úeva02_base_patch14_448ró   )r  r
  r  rD   rD   rE   r  Õ  s    õr  c                 K   s<   t ddddddddddd	d
�}tdd| it |f|Ž—Ž}|S )Nr   r  r  r  r†   r  FTr  r  Úeva02_large_patch14_448ró   )r  r
  r  rD   rD   rE   r  è  s    õr  c              
   K   s:   t ddddd| dd¡d�}tdd
| it |f|Ž—Ž}|S )zB EVA-g CLIP model (only difference from non-CLIP is the pooling)  r  r  r  r†   r  r�   Útoken)rŒ   r�   r‘   r$   rl   r�   Úeva_giant_patch14_clip_224ró   )r   ©r«   Úpoprõ   r  rD   rD   rE   r   û  s        
þr   c                 K   sH   t dddddddddddd| d	d
¡d�}tdd| it |f|Ž—Ž}|S )zU A EVA-CLIP specific variant that adds additional attn scale layernorm to eva02_base r…   r†   r‰   rŠ   Fr  Tr  r�   r  )r‹   rŒ   r�   r‘   r$   r&   rl   rm   rn   ro   rš   rž   r�   Úeva02_base_patch16_clip_224ró   )r#  r!  r  rD   rD   rE   r#    s"    
ór#  c                 K   sH   t dddddddddddd	| d
d¡d�}tdd| it |f|Ž—Ž}|S )úV A EVA-CLIP specific variant that adds additional attn scale layernorm to eva02_large r…   r  r  r  r†   r  FTr  r�   r  ©r‹   rŒ   r�   r‘   r$   rl   r&   rm   rn   ro   rš   rž   r�   Úeva02_large_patch14_clip_224ró   )r&  r!  r  rD   rD   rE   r&    s"    
ór&  c                 K   sH   t dddddddddddd	| d
d¡d�}tdd| it |f|Ž—Ž}|S )r$  rý   r  r  r  r†   r  FTr  r�   r  r%  Úeva02_large_patch14_clip_336ró   )r'  r!  r  rD   rD   rE   r'  1  s"    
ór'  c                 K   s>   t ddddddd| dd	¡d
�}tdd| it |f|Ž—Ž}|S )zD A EVA-CLIP specific variant that uses residual post-norm in blocks r…   r  i   é@   r†   gI’$I’$!@Tr�   r  )r‹   rŒ   r�   r‘   r$   rl   r›   r�   Úeva02_enormous_patch14_clip_224ró   )r)  r!  r  rD   rD   rE   r)  G  s    
ø
r)  )rÜ   T)F)rà   )F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)Cr„   r½   Útypingr   r   r   r   r9   Ztorch.nnr2   Ztorch.nn.functionalZ
functionalrR   Ztorch.utils.checkpointr   Z	timm.datar   r   r	   r
   Ztimm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Z_builderr   Ú	_registryr   r   Ú__all__ÚModuler   ri   rƒ   r   rð   rõ   rû   Zdefault_cfgsr	  r  r  r  r  r  r  r  r  r  r  r   r#  r&  r'  r)  rD   rD   rD   rE   Ú<module>   sž  De`Z    ü
O

þ  ý   ü   ü  ý  ý  ý ý ý ý ý ý   ý   ý   ýýýýýýüüüü ûüüþ€ ä %	