U
    ¯Ê¹j×c  ã                   @   s‚  d Z ddlZddlmZmZmZmZ ddlZddlm	Z	 ddl
m	  mZ ddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddlmZmZmZmZ dd	lmZ dd
l m!Z!m"Z" ddl#m$Z$ dgZ%ee&e&f ej'dœdd„Z(G dd„ de	j)ƒZ*G dd„ de	j)ƒZ+G dd„ de	j)ƒZ,G dd„ de	j)ƒZ-d=dd„Z.e!e.dd�e.dddd�e.ddd �e.dd�e.dddd�e.dd!dd�e.ddd �e.deed"�e.deed"�e.ddeed#�e.dd$eed%�e.dd$eed%�e.ddeed#�d&œƒZ/d>d)d*„Z0d?d,d-„Z1e"d@e-d.œd/d0„ƒZ2e"dAe-d.œd1d2„ƒZ3e"dBe-d.œd3d4„ƒZ4e"dCe-d.œd5d6„ƒZ5e"dDe-d.œd7d8„ƒZ6e"dEe-d.œd9d:„ƒZ7e"dFe-d.œd;d<„ƒZ8dS )Gaé   BEiT: BERT Pre-Training of Image Transformers (https://arxiv.org/abs/2106.08254)

Model from official source: https://github.com/microsoft/unilm/tree/master/beit

@inproceedings{beit,
title={{BEiT}: {BERT} Pre-Training of Image Transformers},
author={Hangbo Bao and Li Dong and Songhao Piao and Furu Wei},
booktitle={International Conference on Learning Representations},
year={2022},
url={https://openreview.net/forum?id=p-BhZSz59o4}
}

BEiT-v2 from https://github.com/microsoft/unilm/tree/master/beit2

@article{beitv2,
title={{BEiT v2}: Masked Image Modeling with Vector-Quantized Visual Tokenizers},
author={Zhiliang Peng and Li Dong and Hangbo Bao and Qixiang Ye and Furu Wei},
year={2022},
eprint={2208.06366},
archivePrefix={arXiv},
primaryClass={cs.CV}
}

At this point only the 1k fine-tuned classification weights and model configs have been added,
see original source above for pre-training models and procedure.

Modifications by / Copyright 2021 Ross Wightman, original copyrights below
é    N)ÚCallableÚOptionalÚTupleÚUnion)Ú
checkpoint)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Ú
PatchEmbedÚMlpÚSwiGLUÚ	LayerNormÚDropPathÚtrunc_normal_Úuse_fused_attn)Úresample_patch_embedÚresample_abs_pos_embedÚresize_rel_pos_bias_tableÚndgridé   )Úbuild_model_with_cfg)Úgenerate_default_cfgsÚregister_model)Úcheckpoint_filter_fnÚBeit)Úwindow_sizeÚreturnc                 C   s„  d| d  d d| d  d  d }| d | d  }t  tt  | d ¡t  | d ¡ƒ¡}t  |d¡}|d d …d d …d f |d d …d d d …f  }| ddd¡ ¡ }|d d …d d …df  | d d 7  < |d d …d d …df  | d d 7  < |d d …d d …df  d| d  d 9  < t j|d fd |jd�}| 	d¡|dd …dd …f< |d |ddd …f< |d |dd …df< |d |d< |S )Né   r   r   é   )ÚsizeÚdtypeéÿÿÿÿ)r   r   )
ÚtorchÚstackr   ZarangeÚflattenÚpermuteÚ
contiguousÚzerosr   Úsum)r   Únum_relative_distanceÚwindow_areaZcoordsZcoords_flattenZrelative_coordsÚrelative_position_index© r+   úO/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/beit.pyÚgen_relative_position_index=   s    $$,&&*r-   c                
       sp   e Zd ZU ejje ed< deeee	e	e
eeef  e
e dœ‡ fdd„Zd	d
„ Zde
ej dœdd„Z‡  ZS )Ú	AttentionÚ
fused_attné   Fç        N)ÚdimÚ	num_headsÚqkv_biasÚ	attn_dropÚ	proj_dropr   Úattn_head_dimc           
         sB  t ƒ  ¡  || _|| }|d k	r$|}|| j }	|d | _tƒ | _tj||	d dd�| _|r–t 	t
 |	¡¡| _| jdt
 |	¡dd� t 	t
 |	¡¡| _nd | _d | _d | _|�r|| _d|d  d	 d|d	  d	  d | _t 	t
 | j|¡¡| _| jd
t|ƒdd� nd | _d | _d | _t |¡| _t |	|¡| _t |¡| _d S )Ng      à¿r   F)ÚbiasÚk_bias)Ú
persistentr   r   r   r*   )ÚsuperÚ__init__r3   Úscaler   r/   ÚnnÚLinearÚqkvÚ	Parameterr!   r&   Úq_biasÚregister_bufferÚv_biasr9   r   r(   Úrelative_position_bias_tabler-   r*   ÚDropoutr5   Úprojr6   )
Úselfr2   r3   r4   r5   r6   r   r7   Zhead_dimZall_head_dim©Ú	__class__r+   r,   r<   T   s:    



&ÿzAttention.__init__c                 C   s`   | j | j d¡  | jd | jd  d | jd | jd  d d¡}| ddd¡ ¡ }| d¡S )Nr    r   r   r   )rE   r*   Úviewr   r$   r%   Z	unsqueeze©rH   Zrelative_position_biasr+   r+   r,   Ú_get_rel_pos_bias€   s    
ÿ ýzAttention._get_rel_pos_bias©Úshared_rel_pos_biasc                 C   sd  |j \}}}| jd k	r,t | j| j| jf¡nd }tj|| jj	|d�}| 
||d| jd¡ ddddd¡}| d¡\}}	}
| jrÔd }| jd k	r¤|  ¡ }|d k	r°|| }n|d k	r°|}tj||	|
|| jrÊ| jjndd	�}nb|| j }||	 d
d¡ }| jd k	�r||  ¡  }|d k	�r|| }|jdd�}|  |¡}||
 }| dd¡ 
|||¡}|  |¡}|  |¡}|S )N)ÚinputÚweightr8   r   r    r   r   r   é   r1   )Z	attn_maskZ	dropout_péþÿÿÿ©r2   )ÚshaperB   r!   Úcatr9   rD   ÚFZlinearr@   rQ   Zreshaper3   r$   Zunbindr/   rE   rM   Zscaled_dot_product_attentionZtrainingr5   Úpr=   Z	transposeZsoftmaxrG   r6   )rH   ÚxrO   ÚBÚNÚCr4   r@   ÚqÚkÚvÚrel_pos_biasÚattnr+   r+   r,   Úforwardˆ   sB    $"

  ý




zAttention.forward)r0   Fr1   r1   NN)N)Ú__name__Ú
__module__Ú__qualname__r!   ÚjitÚFinalÚboolÚ__annotations__ÚintÚfloatr   r   r<   rM   ÚTensorrb   Ú__classcell__r+   r+   rI   r,   r.   Q   s$   
      øø,r.   c                       s‚   e Zd Zddddddddejeddfeeeeeeeeee	e e
e
e	eeef  e	e dœ‡ fdd„Zde	ej dœd	d
„Z‡  ZS )ÚBlockFç      @r1   N)r2   r3   r4   Ú	mlp_ratioÚ	scale_mlpÚ
swiglu_mlpr6   r5   Ú	drop_pathÚinit_valuesÚ	act_layerÚ
norm_layerr   r7   c              	      sø   t ƒ  ¡  ||ƒ| _t|||||||d�| _|	dkr<t|	ƒnt ¡ | _||ƒ| _	|rxt
|t|| ƒ|rj|nd |d�| _n$t|t|| ƒ||r�|nd |d�| _|	dkr¬t|	ƒnt ¡ | _|
rèt |
t |¡ ¡| _t |
t |¡ ¡| _nd\| _| _d S )N)r3   r4   r5   r6   r   r7   r1   )Úin_featuresÚhidden_featuresrv   Údrop)rw   rx   ru   rv   ry   )NN)r;   r<   Únorm1r.   ra   r   r>   ÚIdentityÚ
drop_path1Únorm2r   rj   Úmlpr
   Ú
drop_path2rA   r!   ZonesÚgamma_1Úgamma_2)rH   r2   r3   r4   rp   rq   rr   r6   r5   rs   rt   ru   rv   r   r7   rI   r+   r,   r<   ³   s@    

ù



ü


ûzBlock.__init__rN   c              	   C   sŒ   | j d krD||  | j|  |¡|d�¡ }||  |  |  |¡¡¡ }nD||  | j | j|  |¡|d� ¡ }||  | j|  |  |¡¡ ¡ }|S )NrN   )r€   r|   ra   rz   r   r~   r}   r�   )rH   rY   rO   r+   r+   r,   rb   ê   s    
$ zBlock.forward)N)rc   rd   re   r>   ZGELUr   rj   rh   rk   r   r   r   r<   r!   rl   rb   rm   r+   r+   rI   r,   rn   ±   s:   ññ7rn   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚRelativePositionBiasc                    sn   t ƒ  ¡  || _|d |d  | _d|d  d d|d  d  d }t t ||¡¡| _|  	dt
|ƒ¡ d S )Nr   r   r   r   r*   )r;   r<   r   r)   r>   rA   r!   r&   rE   rC   r-   )rH   r   r3   r(   rI   r+   r,   r<   ö   s    
$zRelativePositionBias.__init__c                 C   s:   | j | j d¡  | jd | jd d¡}| ddd¡ ¡ S )Nr    r   r   r   )rE   r*   rK   r)   r$   r%   rL   r+   r+   r,   rb   ÿ   s      ÿzRelativePositionBias.forward)rc   rd   re   r<   rb   rm   r+   r+   rI   r,   r‚   ô   s   	r‚   c                       s$  e Zd ZdZddddddddd	d
dddddddedd	dddfeeeeef f eeeeef f eeeeeee	e
e	e	e
e
e
e
e
eee
 e	e	e	e
dœ‡ fdd„Zdd„ Zdd„ Zejjdd„ ƒZejjd'dd„ƒZejjd(dd„ƒZejjdd„ ƒZd)dd„Zd d!„ Zd*e	d"œd#d$„Zd%d&„ Z‡  ZS )+r   zI Vision Transformer with support for patch or hybrid CNN input stage
    éà   é   r   éè  Úavgé   é   Tro   Fr1   Ngü©ñÒMbP?)Úimg_sizeÚ
patch_sizeÚin_chansÚnum_classesÚglobal_poolÚ	embed_dimÚdepthr3   r4   rp   rr   rq   Ú	drop_rateÚpos_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_raterv   rt   Úuse_abs_pos_embÚuse_rel_pos_biasÚuse_shared_rel_pos_biasÚhead_init_scalec                    sà  t ƒ  ¡  |ˆ
_|ˆ
_ˆ ˆ
_ˆ
_dˆ
_dˆ
_t|||ˆd�ˆ
_	ˆ
j	j
}t t ddˆ¡¡ˆ
_|rzt t d|d ˆ¡¡nd ˆ
_tj|d�ˆ
_|r¦tˆ
j	jˆd�ˆ
_nd ˆ
_dd„ t d||¡D ƒ‰t ‡ ‡‡‡‡‡‡‡‡‡	‡
‡‡fd	d„t|ƒD ƒ¡ˆ
_ˆ
jd
k}|�rt ¡ nˆˆƒˆ
_|�r*ˆˆƒnt ¡ ˆ
_t |¡ˆ
_|dk�rVt ˆ|¡nt ¡ ˆ
_ˆ
  ˆ
j!¡ ˆ
jd k	�r†t"ˆ
jdd� t"ˆ
jdd� ˆ
 #¡  t$ˆ
jtjƒ�rÜt"ˆ
jj%dd� ˆ
jj%j& '|¡ ˆ
jj(j& '|¡ d S )Nr   F)r‰   rŠ   r‹   rŽ   )rX   )r   r3   c                 S   s   g | ]}|  ¡ ‘qS r+   )Úitem)Ú.0rY   r+   r+   r,   Ú
<listcomp>?  s     z!Beit.__init__.<locals>.<listcomp>r   c                    s<   g | ]4}t ˆˆˆˆˆ	ˆˆˆ ˆ| ˆˆˆr0ˆ
jjnd d�‘qS )N)r2   r3   r4   rp   rq   rr   r6   r5   rs   rv   rt   r   )rn   Úpatch_embedÚ	grid_size)rš   Úi©r“   ZdprrŽ   rt   rp   rv   r3   r’   r4   rq   rH   rr   r–   r+   r,   r›   @  s   òôr†   ç{®Gáz”?©Ústd))r;   r<   rŒ   r�   Znum_featuresrŽ   Únum_prefix_tokensÚgrad_checkpointingr	   rœ   Únum_patchesr>   rA   r!   r&   Ú	cls_tokenÚ	pos_embedrF   Úpos_dropr‚   r�   r`   ZlinspaceZ
ModuleListÚrangeÚblocksr{   ÚnormÚfc_normÚ	head_dropr?   ÚheadÚapplyÚ_init_weightsr   Úfix_init_weightÚ
isinstancerQ   ÚdataZmul_r8   )rH   r‰   rŠ   r‹   rŒ   r�   rŽ   r�   r3   r4   rp   rr   rq   r�   r‘   r’   r“   r”   rv   rt   r•   r–   r—   r˜   r¥   Zuse_fc_normrI   rŸ   r,   r<   	  sP    
ü"þ
&ñ

 zBeit.__init__c                 C   sL   dd„ }t | jƒD ]4\}}||jjjj|d ƒ ||jjjj|d ƒ qd S )Nc                 S   s   |   t d| ¡¡ d S )Ng       @)Zdiv_ÚmathÚsqrt)ÚparamÚlayer_idr+   r+   r,   Úrescalec  s    z%Beit.fix_init_weight.<locals>.rescaler   )Ú	enumeraterª   ra   rG   rQ   r³   r~   Zfc2)rH   r¸   r·   Úlayerr+   r+   r,   r±   b  s    zBeit.fix_init_weightc                 C   sr   t |tjƒrBt|jdd� t |tjƒrn|jd k	rntj |jd¡ n,t |tjƒrntj |jd¡ tj |jd¡ d S )Nr    r¡   r   ç      ð?)	r²   r>   r?   r   rQ   r8   ÚinitZ	constant_r   )rH   Úmr+   r+   r,   r°   j  s    zBeit._init_weightsc                 C   s0   ddh}|   ¡ D ]\}}d|kr| |¡ q|S )Nr§   r¦   rE   )Znamed_parametersÚadd)rH   ZnwdÚnÚ_r+   r+   r,   Úno_weight_decays  s
    zBeit.no_weight_decayc                 C   s
   || _ d S ©N)r¤   )rH   Úenabler+   r+   r,   Úset_grad_checkpointing{  s    zBeit.set_grad_checkpointingc                 C   s   t dddgd�}|S )Nz-^cls_token|pos_embed|patch_embed|rel_pos_bias)z^blocks\.(\d+)N)z^norm)iŸ† )Ústemrª   )Údict)rH   ZcoarseZmatcherr+   r+   r,   Úgroup_matcher  s
    þzBeit.group_matcherc                 C   s   | j S rÂ   )r®   )rH   r+   r+   r,   Úget_classifier‡  s    zBeit.get_classifierc                 C   s8   || _ |d k	r|| _|dkr*t | j|¡nt ¡ | _d S )Nr   )rŒ   r�   r>   r?   rŽ   r{   r®   )rH   rŒ   r�   r+   r+   r,   Úreset_classifier‹  s    zBeit.reset_classifierc                 C   s¨   |   |¡}tj| j |jd dd¡|fdd�}| jd k	rB|| j }|  |¡}| jd k	r^|  ¡ nd }| j	D ]0}| j
rŒtj ¡ sŒt|||d�}qh|||d�}qh|  |¡}|S )Nr   r    r   rT   rN   )rœ   r!   rV   r¦   ÚexpandrU   r§   r¨   r`   rª   r¤   rf   Zis_scriptingr   r«   )rH   rY   r`   Zblkr+   r+   r,   Úforward_features‘  s    
$




zBeit.forward_features)Ú
pre_logitsc                 C   sd   | j r>| j dkr.|d d …| jd …f jdd�n|d d …df }|  |¡}|  |¡}|rZ|S |  |¡S )Nr†   r   rT   r   )r�   r£   Úmeanr¬   r­   r®   )rH   rY   rÌ   r+   r+   r,   Úforward_head¡  s
    8

zBeit.forward_headc                 C   s   |   |¡}|  |¡}|S rÂ   )rË   rÎ   )rH   rY   r+   r+   r,   rb   ¨  s    

zBeit.forward)T)F)N)F)rc   rd   re   Ú__doc__r   r   rj   r   Ústrrh   rk   r   r   r<   r±   r°   r!   rf   ÚignorerÁ   rÄ   rÇ   rÈ   rÉ   rË   rÎ   rb   rm   r+   r+   rI   r,   r     s~   èèY	


Ú c                 K   s    | ddd dddddddd	œ|–S )
Nr…   )r   rƒ   rƒ   gÍÌÌÌÌÌì?ÚbicubicT)ç      à?rÔ   rÔ   zpatch_embed.projr®   )ÚurlrŒ   Ú
input_sizeZ	pool_sizeÚcrop_pctÚinterpolationZfixed_input_sizerÍ   r¢   Z
first_convÚ
classifierr+   )rÕ   Úkwargsr+   r+   r,   Ú_cfg®  s          ûúrÛ   ztimm/)Ú	hf_hub_id)r   é€  rÝ   r»   )rÜ   rÖ   r×   iQU  )rÜ   rŒ   )r   é   rÞ   )rÜ   rÍ   r¢   )rÜ   rŒ   rÍ   r¢   gffffffî?)rÜ   r×   rÍ   r¢   )z)beit_base_patch16_224.in22k_ft_in22k_in1kz)beit_base_patch16_384.in22k_ft_in22k_in1kz$beit_base_patch16_224.in22k_ft_in22kz*beit_large_patch16_224.in22k_ft_in22k_in1kz*beit_large_patch16_384.in22k_ft_in22k_in1kz*beit_large_patch16_512.in22k_ft_in22k_in1kz%beit_large_patch16_224.in22k_ft_in22kz*beitv2_base_patch16_224.in1k_ft_in22k_in1kz$beitv2_base_patch16_224.in1k_ft_in1kz%beitv2_base_patch16_224.in1k_ft_in22kz+beitv2_large_patch16_224.in1k_ft_in22k_in1kz%beitv2_large_patch16_224.in1k_ft_in1kz&beitv2_large_patch16_224.in1k_ft_in22krÓ   Tc              	   C   s(  |   d| ¡} |   d| ¡} i }|  ¡ D ]þ\}}d|kr6q$d|kr†|jjjj\}}}	}
|jd |
ksn|jd |	kr„t||	|
f||dd�}n”|d	krÂ|jd
 |jjd
 krÂd
}t||jj	|||dd�}nX| 
d¡�r| |d d… ¡}|j|jjk�s|jd |jd
 k�rt||j|jjd�}|||< q$|S )NÚmodelÚmoduler*   zpatch_embed.proj.weightr    rS   T)rØ   Ú	antialiasÚverboser§   r   )Únew_sizer£   rØ   rá   râ   rE   iãÿÿÿr   )Znew_window_sizeZnew_bias_shape)ÚgetÚitemsrœ   rG   rQ   rU   r   r§   r   r�   ÚendswithZget_submodulerE   r   r   )Z
state_dictrß   rØ   rá   Zout_dictr^   r_   ÚOÚIÚHÚWr£   r½   r+   r+   r,   Ú_beit_checkpoint_filter_fnû  sH    ûú&ý
rë   Fc                 K   s0   |  dd ¡rtdƒ‚tt| |fdti|—Ž}|S )NZfeatures_onlyz.features_only not implemented for BEiT models.Zpretrained_filter_fn)rä   ÚRuntimeErrorr   r   rë   )ÚvariantÚ
pretrainedrÚ   rß   r+   r+   r,   Ú_create_beit%  s      ÿýürï   )r   c              
   K   s6   t ddddddddd�}tdd
| it |f|Ž—Ž}|S )Nr„   r‡   rˆ   rR   FTçš™™™™™¹?©rŠ   rŽ   r�   r3   rp   r•   r–   rt   Úbeit_base_patch16_224rî   )rò   ©rÆ   rï   ©rî   rÚ   Z
model_argsrß   r+   r+   r,   rò   1  s          þrò   c              
   K   s6   t ddddddddd�}tdd
| it |f|Ž—Ž}|S )NrÝ   r„   r‡   rˆ   FTrð   ©r‰   rŠ   rŽ   r�   r3   r•   r–   rt   Úbeit_base_patch16_384rî   )rö   ró   rô   r+   r+   r,   rö   :  s          þrö   c              	   K   s4   t dddddddd�}td
d	| it |f|Ž—Ž}|S )Nr„   é   é   FTçñhãˆµøä>©rŠ   rŽ   r�   r3   r•   r–   rt   Úbeit_large_patch16_224rî   )rû   ró   rô   r+   r+   r,   rû   C  s         þrû   c              
   K   s6   t ddddddddd�}tdd
| it |f|Ž—Ž}|S )NrÝ   r„   r÷   rø   FTrù   rõ   Úbeit_large_patch16_384rî   )rü   ró   rô   r+   r+   r,   rü   L  s          þrü   c              
   K   s6   t ddddddddd�}tdd
| it |f|Ž—Ž}|S )NrÞ   r„   r÷   rø   FTrù   rõ   Úbeit_large_patch16_512rî   )rý   ró   rô   r+   r+   r,   rý   U  s          þrý   c              
   K   s6   t ddddddddd�}tdd
| it |f|Ž—Ž}|S )Nr„   r‡   rˆ   rR   FTrù   rñ   Úbeitv2_base_patch16_224rî   )rþ   ró   rô   r+   r+   r,   rþ   ^  s          þrþ   c              	   K   s4   t dddddddd�}td
d	| it |f|Ž—Ž}|S )Nr„   r÷   rø   FTrù   rú   Úbeitv2_large_patch16_224rî   )rÿ   ró   rô   r+   r+   r,   rÿ   g  s         þrÿ   )rÒ   )rÓ   T)F)F)F)F)F)F)F)F)9rÏ   r´   Útypingr   r   r   r   r!   Ztorch.nnr>   Ztorch.nn.functionalZ
functionalrW   Ztorch.utils.checkpointr   Z	timm.datar   r   Ztimm.layersr	   r
   r   r   r   r   r   r   r   r   r   Z_builderr   Ú	_registryr   r   Zvision_transformerr   Ú__all__rj   rl   r-   ÚModuler.   rn   r‚   r   rÛ   Zdefault_cfgsrë   rï   rò   rö   rû   rü   rý   rþ   rÿ   r+   r+   r+   r,   Ú<module>   sÊ   ($`C *
þ ýýþ ý ýý ý ý  ý  ý  ý  ýÆB
*
