U
    ¯Ê¹jÅ;  ã                   @   sL  d Z ddlmZ ddlZddlmZ ddlmZmZ ddl	m
Z
mZmZmZmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ dgZeG dd„ dejƒƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZd#dd„Zd$dd„Zeedd�edd�edd�dœƒZ ed%edœdd„ƒZ!ed&edœdd „ƒZ"ed'edœd!d"„ƒZ#dS )(a   ConViT Model

@article{d2021convit,
  title={ConViT: Improving Vision Transformers with Soft Convolutional Inductive Biases},
  author={d'Ascoli, St{'e}phane and Touvron, Hugo and Leavitt, Matthew and Morcos, Ari and Biroli, Giulio and Sagun, Levent},
  journal={arXiv preprint arXiv:2103.10697},
  year={2021}
}

Paper link: https://arxiv.org/abs/2103.10697
Original code: https://github.com/facebookresearch/convit, original copyright below

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
é    )ÚpartialN©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚtrunc_normal_Ú
PatchEmbedÚMlpÚ	LayerNormé   )Úbuild_model_with_cfg)Úregister_notrace_module)Úregister_modelÚgenerate_default_cfgs)ÚHybridEmbedÚConVitc                       sR   e Zd Zd‡ fdd„	Zdd„ Zd	d
„ Zddd„Zdd„ Zee	j
dœdd„Z‡  ZS )ÚGPSAé   Fç        ç      ð?c                    s´   t ƒ  ¡  || _|| _|| }|d | _|| _tj||d |d�| _tj|||d�| _	t 
|¡| _t ||¡| _t d|¡| _t 
|¡| _t t | j¡¡| _t dddd¡| _d S )Nç      à¿é   ©Úbiasé   r   )ÚsuperÚ__init__Ú	num_headsÚdimÚscaleÚlocality_strengthÚnnÚLinearÚqkÚvÚDropoutÚ	attn_dropÚprojÚpos_projÚ	proj_dropÚ	ParameterÚtorchZonesÚgating_paramÚzerosÚrel_indices)Úselfr   r   Úqkv_biasr&   r)   r    Úhead_dim©Ú	__class__© úQ/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/convit.pyr   +   s    	

zGPSA.__init__c                 C   s˜   |j \}}}| jd ks&| jj d |kr2|  |¡| _|  |¡}|  |¡ ||| j|| j ¡ dddd¡}||  dd¡ |||¡}|  	|¡}|  
|¡}|S )Nr   r   r   r   )Úshaper.   Úget_rel_indicesÚget_attentionr$   Úreshaper   ÚpermuteÚ	transposer'   r)   )r/   ÚxÚBÚNÚCÚattnr$   r4   r4   r5   ÚforwardE   s    
*

zGPSA.forwardc                 C   sø   |j \}}}|  |¡ ||d| j|| j ¡ ddddd¡}|d |d  }}| j |ddd¡}|  |¡ dddd¡}|| dd¡ | j	 }	|	j
dd�}	|j
dd�}| j dddd¡}
d	t |
¡ |	 t |
¡|  }||jdd� d¡ }|  |¡}|S )
Nr   r   r   r   é   éÿÿÿÿéþÿÿÿ©r   r   )r6   r#   r9   r   r:   r.   Úexpandr(   r;   r   Úsoftmaxr,   Úviewr+   ZsigmoidÚsumÚ	unsqueezer&   )r/   r<   r=   r>   r?   r#   ÚqÚkZ	pos_scoreZpatch_scoreZgatingr@   r4   r4   r5   r8   P   s    . 
zGPSA.get_attentionc                 C   s^   |   |¡ d¡}| j ¡ d d …d d …df d }t d||f¡| d¡ }|rV||fS |S d S )Nr   rC   ç      à?ú	nm,hnm->h)r8   Úmeanr.   Zsqueezer+   ÚeinsumÚsize)r/   r<   Ú
return_mapÚattn_mapÚ	distancesÚdistr4   r4   r5   Úget_attention_map`   s     zGPSA.get_attention_mapc                 C   sØ   | j jj t | j¡¡ d}t| jd ƒ}|d dkrB|d d n|d }t	|ƒD ]l}t	|ƒD ]^}|||  }d| j
jj|df< d||  | | j
jj|df< d||  | | j
jj|df< q^qR| j
j j| j9  _d S )Nr   rM   r   r   rC   )r$   ÚweightÚdataZcopy_r+   Úeyer   Úintr   Úranger(   r    )r/   Zlocality_distanceZkernel_sizeÚcenterZh1Zh2Úpositionr4   r4   r5   Ú
local_initi   s     "zGPSA.local_init)Únum_patchesÚreturnc           	      C   sî   t |d ƒ}t d||d¡}t |¡ dd¡t |¡ dd¡ }| ||¡}|j|dd�j|dd�}|d |d  }| d¡|d d …d d …d d …df< | d¡|d d …d d …d d …df< | d¡|d d …d d …d d …df< | jj	j
}| |¡S )NrM   r   r   rC   r   rE   r   )rZ   r+   r-   ÚarangerH   ÚrepeatÚrepeat_interleaverJ   r#   rW   ÚdeviceÚto)	r/   r_   Úimg_sizer.   ÚindÚindxÚindyÚinddrd   r4   r4   r5   r7   w   s    $"""
zGPSA.get_rel_indices)r   Fr   r   r   )F)Ú__name__Ú
__module__Ú__qualname__r   rA   r8   rV   r^   rZ   r+   ZTensorr7   Ú__classcell__r4   r4   r2   r5   r   )   s        ù
	r   c                       s0   e Zd Zd
‡ fdd„	Zddd„Zdd	„ Z‡  ZS )ÚMHSAr   Fr   c                    sb   t ƒ  ¡  || _|| }|d | _tj||d |d�| _t |¡| _t ||¡| _	t |¡| _
d S )Nr   r   r   )r   r   r   r   r!   r"   Úqkvr%   r&   r'   r)   )r/   r   r   r0   r&   r)   r1   r2   r4   r5   r   †   s    

zMHSA.__init__c                 C   s   |j \}}}|  |¡ ||d| j|| j ¡ ddddd¡}|d |d |d   }}}	|| dd¡ | j }
|
jdd� d¡}
t	|d	 ƒ}t
 |¡ dd¡t
 |¡ dd¡ }| ||¡}|j|dd�j|dd�}|d |d  }|d	 }| |j¡}t
 d
||
f¡| }|�r||
fS |S d S )Nr   r   r   r   rB   rD   rC   rE   rM   rN   )r6   rp   r9   r   r:   r;   r   rG   rO   rZ   r+   ra   rH   rb   rc   re   rd   rP   )r/   r<   rR   r=   r>   r?   rp   rK   rL   r$   rS   rf   rg   rh   ri   rj   rT   rU   r4   r4   r5   rV   ˜   s     .$zMHSA.get_attention_mapc           
      C   s¨   |j \}}}|  |¡ ||d| j|| j ¡ ddddd¡}| d¡\}}}|| dd¡ | j }	|	jdd�}	|  	|	¡}	|	|  dd¡ |||¡}|  
|¡}|  |¡}|S )	Nr   r   r   r   rB   rD   rC   rE   )r6   rp   r9   r   r:   Zunbindr;   r   rG   r&   r'   r)   )
r/   r<   r=   r>   r?   rp   rK   rL   r$   r@   r4   r4   r5   rA   ­   s    .


zMHSA.forward)r   Fr   r   )F)rk   rl   rm   r   rV   rA   rn   r4   r4   r2   r5   ro   …   s       ú
ro   c                	       s:   e Zd Zdddddejeddf	‡ fdd„	Zdd	„ Z‡  ZS )
ÚBlockç      @Fr   Tr   c                    s’   t ƒ  ¡  |	|ƒ| _|
| _| jr8t||||||d�| _nt|||||d�| _|dkr\t|ƒnt 	¡ | _
|	|ƒ| _t|| ƒ}t||||d�| _d S )N)r   r0   r&   r)   r    )r   r0   r&   r)   r   )Zin_featuresZhidden_featuresÚ	act_layerZdrop)r   r   Únorm1Úuse_gpsar   r@   ro   r   r!   ÚIdentityÚ	drop_pathÚnorm2rZ   r	   Úmlp)r/   r   r   Ú	mlp_ratior0   r)   r&   rw   rs   Ú
norm_layerru   r    Zmlp_hidden_dimr2   r4   r5   r   ¾   s8    

ú
	û
üzBlock.__init__c                 C   s8   ||   |  |  |¡¡¡ }||   |  |  |¡¡¡ }|S ©N)rw   r@   rt   ry   rx   ©r/   r<   r4   r4   r5   rA   ê   s    zBlock.forward)	rk   rl   rm   r!   ZGELUr
   r   rA   rn   r4   r4   r2   r5   rq   ¼   s   ô,rq   c                       sÀ   e Zd ZdZddddddddd	d
ddddddedddf‡ fdd„	Zdd„ Zejj	dd„ ƒZ
ejj	d$dd„ƒZejj	d%dd„ƒZejj	dd„ ƒZd&dd„Zdd„ Zd'edœd d!„Zd"d#„ Z‡  ZS )(r   zI Vision Transformer with support for patch or hybrid CNN input stage
    éà   é   r   éè  Útokeni   é   rr   Fr   Nr   Tc                    s¨  t ƒ  ¡  |dkst‚ˆˆ9 ‰|| _|| _ˆ| _ˆ | _| _ˆ| _|| _	|d k	rdt
|||ˆd�| _nt|||ˆd�| _| jj}|| _t t ddˆ¡¡| _tj|d�| _| j	rÒt t d|ˆ¡¡| _t| jdd� dd	„ t d
||¡D ƒ‰t ‡ ‡‡‡‡‡‡‡‡‡	f
dd	„t|ƒD ƒ¡| _ˆˆƒ| _tˆd
dd�g| _t |¡| _|d
k�rXt ˆ|¡nt  ¡ | _!t| jdd� |  "| j#¡ |  $¡ D ]\}}t%|dƒ�r„| &¡  �q„d S )N)Ú Úavgr�   )rf   Úin_chansÚ	embed_dim)rf   Ú
patch_sizer…   r†   r   )Úpç{®Gáz”?©Ústdc                 S   s   g | ]}|  ¡ ‘qS r4   )Úitem)Ú.0r<   r4   r4   r5   Ú
<listcomp>)  s     z#ConVit.__init__.<locals>.<listcomp>r   c                    s0   g | ](}t ˆˆˆˆ	ˆˆ ˆ| ˆ|ˆk ˆd �
‘qS ))
r   r   rz   r0   r)   r&   rw   r{   ru   r    )rq   )r�   Úi©
Úattn_drop_rateZdprr†   Úlocal_up_to_layerr    rz   r{   r   Úproj_drop_rater0   r4   r5   rŽ   *  s   õöÚhead)Znum_chsZ	reductionÚmoduler^   )'r   r   ÚAssertionErrorÚnum_classesÚglobal_poolr’   Znum_featuresr†   r    Úuse_pos_embedr   Úpatch_embedr   r_   r!   r*   r+   r-   Ú	cls_tokenr%   Úpos_dropÚ	pos_embedr   ZlinspaceZ
ModuleListr[   ÚblocksÚnormÚdictZfeature_infoÚ	head_dropr"   rv   r”   ÚapplyÚ_init_weightsZnamed_modulesÚhasattrr^   )r/   rf   r‡   r…   r—   r˜   r†   Údepthr   rz   r0   Z	drop_rateZpos_drop_rater“   r‘   Zdrop_path_rateZhybrid_backboner{   r’   r    r™   r_   ÚnÚmr2   r�   r5   r   ô   sT    
   ÿ
ü ô

 zConVit.__init__c                 C   sr   t |tjƒrBt|jdd� t |tjƒrn|jd k	rntj |jd¡ n,t |tjƒrntj |jd¡ tj |jd¡ d S )Nr‰   rŠ   r   r   )	Ú
isinstancer!   r"   r   rW   r   ÚinitZ	constant_r
   )r/   r§   r4   r4   r5   r£   D  s    zConVit._init_weightsc                 C   s   ddhS )Nr�   r›   r4   ©r/   r4   r4   r5   Úno_weight_decayM  s    zConVit.no_weight_decayc                 C   s   t dddgd�S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )Ústemrž   )r    )r/   Zcoarser4   r4   r5   Úgroup_matcherQ  s    þzConVit.group_matcherc                 C   s   |rt dƒ‚d S )Nz$gradient checkpointing not supported)r–   )r/   Úenabler4   r4   r5   Úset_grad_checkpointingX  s    zConVit.set_grad_checkpointingc                 C   s   | j S r|   )r”   rª   r4   r4   r5   Úget_classifier\  s    zConVit.get_classifierc                 C   sD   || _ |d k	r |dkst‚|| _|dkr6t | j|¡nt ¡ | _d S )N)rƒ   r�   r„   r   )r—   r–   r˜   r!   r"   r†   rv   r”   )r/   r—   r˜   r4   r4   r5   Úreset_classifier`  s
    zConVit.reset_classifierc                 C   s€   |   |¡}| jr|| j }|  |¡}| j |jd dd¡}t| jƒD ],\}}|| j	krht
j||fdd�}||ƒ}qD|  |¡}|S )Nr   rC   r   rE   )rš   r™   r�   rœ   r›   rF   r6   Ú	enumeraterž   r’   r+   ÚcatrŸ   )r/   r<   Z
cls_tokensÚuZblkr4   r4   r5   Úforward_featuresg  s    





zConVit.forward_features)Ú
pre_logitsc                 C   sX   | j r<| j dkr,|d d …dd …f jdd�n|d d …df }|  |¡}|rN|S |  |¡S )Nr„   r   rE   r   )r˜   rO   r¡   r”   )r/   r<   r¶   r4   r4   r5   Úforward_headt  s    6
zConVit.forward_headc                 C   s   |   |¡}|  |¡}|S r|   )rµ   r·   r}   r4   r4   r5   rA   z  s    

zConVit.forward)F)T)N)F)rk   rl   rm   Ú__doc__r
   r   r£   r+   ZjitÚignorer«   r­   r¯   r°   r±   rµ   Úboolr·   rA   rn   r4   r4   r2   r5   r   ð   sF   ëP	


Fc                 K   s$   |  dd ¡rtdƒ‚tt| |f|ŽS )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.)ÚgetÚRuntimeErrorr   r   )ÚvariantÚ
pretrainedÚkwargsr4   r4   r5   Ú_create_convit€  s    rÀ   rƒ   c              
   K   s   | ddd t tddddœ	|–S )Nr€   )r   r~   r~   Tzpatch_embed.projr”   )	Úurlr—   Z
input_sizeZ	pool_sizerO   r‹   Zfixed_input_sizeZ
first_convÚ
classifierr   )rÁ   r¿   r4   r4   r5   Ú_cfg‡  s         üûrÃ   ztimm/)Z	hf_hub_id)zconvit_tiny.fb_in1kzconvit_small.fb_in1kzconvit_base.fb_in1k)r`   c                 K   s0   t ddddd�}tf d| dœt |f|Ž—Ž}|S )Né
   r   é0   rB   ©r’   r    r†   r   Úconvit_tiny©r½   r¾   ©r    rÀ   ©r¾   r¿   Z
model_argsÚmodelr4   r4   r5   rÇ   ™  s       ÿrÇ   c                 K   s0   t ddddd�}tf d| dœt |f|Ž—Ž}|S )NrÄ   r   rÅ   é	   rÆ   Úconvit_smallrÈ   rÉ   rÊ   r4   r4   r5   rÍ   ¡  s       ÿrÍ   c                 K   s0   t ddddd�}tf d| dœt |f|Ž—Ž}|S )NrÄ   r   rÅ   r   rÆ   Úconvit_baserÈ   rÉ   rÊ   r4   r4   r5   rÎ   ©  s       ÿrÎ   )F)rƒ   )F)F)F)$r¸   Ú	functoolsr   r+   Ztorch.nnr!   Z	timm.datar   r   Ztimm.layersr   r   r   r	   r
   Z_builderr   Z_features_fxr   Ú	_registryr   r   Zvision_transformer_hybridr   Ú__all__ÚModuler   ro   rq   r   rÀ   rÃ   Zdefault_cfgsrÇ   rÍ   rÎ   r4   r4   r4   r5   Ú<module>   s:   [74 


ü