U
    ¯Ê¹j&J  ã                   @   s  d Z ddlZddlmZ ddlmZmZ ddlmZm	Z	m
Z
mZmZmZmZ ddlmZ ddlmZ ddlmZmZ d	gZG d
d„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd	„ d	ejƒZddd„Zd dd„Zeedd�edd�dœƒZed!edœdd„ƒZed"edœdd„ƒZ dS )#zð Visformer

Paper: Visformer: The Vision-friendly Transformer - https://arxiv.org/abs/2104.12533

From original at https://github.com/danczs/Visformer

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
é    N©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Ú	to_2tupleÚtrunc_normal_ÚDropPathÚ
PatchEmbedÚLayerNorm2dÚcreate_classifierÚuse_fused_attné   )Úbuild_model_with_cfg)Úcheckpoint_seq)Úregister_modelÚgenerate_default_cfgsÚ	Visformerc                       s4   e Zd Zddejdddf‡ fdd„	Zdd„ Z‡  ZS )	Ú
SpatialMlpNç        é   Fc           	   	      sø   t ƒ  ¡  |p|}|p|}t|ƒ}|| _|| _|| _| jrX|dk rP|d d }n|d }|| _|| _tj	||ddddd�| _
|ƒ | _t |d ¡| _| jrÀtj	||ddd| jdd	�| _|ƒ | _nd | _d | _tj	||ddddd�| _t |d ¡| _d S )
Né   é   é   r   r   F©ÚstrideÚpaddingÚbiasé   )r   r   Úgroupsr   )ÚsuperÚ__init__r   Úin_featuresÚout_featuresÚspatial_convÚhidden_featuresÚgroupÚnnÚConv2dÚconv1Úact1ÚDropoutÚdrop1Úconv2Úact2Úconv3Údrop3)	Úselfr    r#   r!   Ú	act_layerÚdropr$   r"   Z
drop_probs©Ú	__class__© úT/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/visformer.pyr      s>    

      ÿ
zSpatialMlp.__init__c                 C   sT   |   |¡}|  |¡}|  |¡}| jd k	r<|  |¡}|  |¡}|  |¡}|  |¡}|S ©N)r'   r(   r*   r+   r,   r-   r.   ©r/   Úxr4   r4   r5   Úforward=   s    







zSpatialMlp.forward)Ú__name__Ú
__module__Ú__qualname__r%   ÚGELUr   r9   Ú__classcell__r4   r4   r2   r5   r      s   ø&r   c                       s8   e Zd ZU ejje ed< d	‡ fdd„	Zdd„ Z	‡  Z
S )
Ú	AttentionÚ
fused_attnr   ç      ð?r   c                    sž   t ƒ  ¡  || _|| _t|| | ƒ}|| _|d | _tdd�| _t	j
||| d ddddd�| _t	 |¡| _t	j
| j| j |ddddd�| _t	 |¡| _d S )	Ng      à¿T)Zexperimentalr   r   r   Fr   )r   r   ÚdimÚ	num_headsÚroundÚhead_dimÚscaler   r@   r%   r&   Úqkvr)   Ú	attn_dropÚprojÚ	proj_drop)r/   rB   rC   Úhead_dim_ratiorH   rJ   rE   r2   r4   r5   r   L   s    

  zAttention.__init__c           
      C   sè   |j \}}}}|  |¡ |d| j| jd¡ ddddd¡}| d¡\}}}| jr€tj	j
j| ¡ | ¡ | ¡ | jrv| jjndd�}n4|| d	d¡ | j }	|	jdd
�}	|  |	¡}	|	| }| dddd¡ |d||¡}|  |¡}|  |¡}|S )Nr   éÿÿÿÿr   r   r   é   r   )Z	dropout_péþÿÿÿ)rB   )ÚshaperG   ZreshaperC   rE   ZpermuteZunbindr@   Útorchr%   Z
functionalZscaled_dot_product_attentionÚ
contiguousZtrainingrH   ÚpZ	transposerF   ZsoftmaxrI   rJ   )
r/   r8   ÚBÚCÚHÚWÚqÚkÚvÚattnr4   r4   r5   r9   Z   s$    *  þ


zAttention.forward)r   rA   r   r   )r:   r;   r<   rP   ÚjitÚFinalÚboolÚ__annotations__r   r9   r>   r4   r4   r2   r5   r?   I   s   
r?   c                
       s<   e Zd Zdddddejedddf
‡ fdd„	Zdd	„ Z‡  ZS )
ÚBlockrA   ç      @r   r   Fc                    s†   t ƒ  ¡  || _|dkr t|ƒnt ¡ | _|r<d | _d | _n|	|ƒ| _t	|||||d�| _|	|ƒ| _
t|t|| ƒ|||
|d�| _d S )Nr   )rC   rK   rH   rJ   )r    r#   r0   r1   r$   r"   )r   r   r"   r   r%   ZIdentityÚ	drop_pathÚnorm1rZ   r?   Únorm2r   ÚintÚmlp)r/   rB   rC   rK   Ú	mlp_ratiorJ   rH   ra   r0   Ú
norm_layerr$   Úattn_disabledr"   r2   r4   r5   r   q   s.    

û

úzBlock.__init__c                 C   sB   | j d k	r$||  |   |  |¡¡¡ }||  |  |  |¡¡¡ }|S r6   )rZ   ra   rb   re   rc   r7   r4   r4   r5   r9   š   s    
zBlock.forward)	r:   r;   r<   r%   r=   r	   r   r9   r>   r4   r4   r2   r5   r_   p   s   ó)r_   c                       s²   e Zd Zddddddddd	d
d
d
d
d
eddddddddf‡ fdd„	Zdd„ Zejjd$dd„ƒZ	ejjd%dd„ƒZ
ejjdd„ ƒZd&dd„Zdd„ Zd'edœd d!„Zd"d#„ Z‡  ZS )(r   éà   é   r   éè  é    é€  é   r   r`   r   Z111TFr   ÚavgNc                    s  t ƒ  ¡  t|ƒ}|| _ˆ| _|| _|| _|| _|| _t	|t
tfƒr^|\| _| _| _t|ƒ}n"|d  | _| _|| j | j | _|| _d| _dd„ t d||¡D ƒ‰| jrÚd | _t|ˆ|ˆ|dd�| _‡fdd„|D ƒ}n¸| jd k�rd | _t|ˆd |ˆd |dd�| _‡fd	d„|D ƒ}ntt tj|| jd
dddd�t | j¡tjdd�¡| _dd„ |D ƒ}t|ˆd | jˆd |dd�| _‡fdd„|D ƒ}| j�rì| j�r¾t tjdˆf|žŽ ¡| _nt tjdˆd f|žŽ ¡| _tj|d�| _ nd | _tj‡ ‡‡‡‡‡‡‡‡	‡
f
dd„t!| jƒD ƒŽ | _"| j�sˆt|ˆd ˆd ˆ|dd�| _#‡fdd„|D ƒ}| j�r€t tjdˆf|žŽ ¡| _$nd | _$nd | _#tj‡ ‡‡‡‡‡‡‡‡	‡
f
dd„t!| j| j| j ƒD ƒŽ | _%| j�s2t|ˆd ˆˆd |dd�| _&‡fdd„|D ƒ}| j�r*t tjdˆd f|žŽ ¡| _'nd | _'nd | _&tj‡ ‡‡‡‡‡‡‡‡	‡
f
dd„t!| j| j |ƒD ƒŽ | _(| j�r~ˆnˆd | _)ˆ| j)ƒ| _*t+| j)| j|d�\}}|| _,t |
¡| _-|| _.| j�rüt/| jdd� | j�süt/| j$dd� t/| j'dd� |  0| j1¡ d S )Nr   Fc                 S   s   g | ]}|  ¡ ‘qS r4   )Úitem©Ú.0r8   r4   r4   r5   Ú
<listcomp>Í   s     z&Visformer.__init__.<locals>.<listcomp>r   )Úimg_sizeÚ
patch_sizeÚin_chansÚ	embed_dimrg   Úflattenc                    s   g | ]}|ˆ  ‘qS r4   r4   rq   ©ru   r4   r5   rs   Ù   s     r   c                    s   g | ]}|ˆ d   ‘qS ©r   r4   rq   ry   r4   r5   rs   å   s     é   r   T)Zinplacec                 S   s   g | ]}|d  ‘qS rz   r4   rq   r4   r4   r5   rs   ì   s     rM   c                    s   g | ]}|ˆ d   ‘qS )rM   r4   rq   ry   r4   r5   rs   õ   s     r   )rR   c                    sB   g | ]:}t ˆd  ˆdˆˆˆ ˆ| ˆˆˆd dkˆ	d dkd�‘qS )r   g      à?r   Ú0Ú1©rB   rC   rK   rf   rJ   rH   ra   rg   r$   rh   r"   ©r_   ©rr   Úi©
Úattn_drop_rateÚ
attn_stageÚdprrw   r$   rf   rg   rC   Úproj_drop_rater"   r4   r5   rs      s   ó

õr   c                    s   g | ]}|ˆ d   ‘qS ©r   r4   rq   ry   r4   r5   rs     s     c                    s>   g | ]6}t ˆˆd ˆˆˆ ˆ| ˆˆˆd dkˆ	d dkd�‘qS )rA   r   r|   r}   r~   r   r€   r‚   r4   r5   rs   "  s   ó

õc                    s   g | ]}|ˆ d   ‘qS r‡   r4   rq   ry   r4   r5   rs   =  s     c                    sB   g | ]:}t ˆd  ˆdˆˆˆ ˆ| ˆˆˆd  dkˆ	d  dkd�‘qS )r   rA   r|   r}   r~   r   r€   r‚   r4   r5   rs   D  s   ó

õ©Z	pool_typeç{®Gáz”?©Ústd)2r   r   r   Únum_classesrw   Úinit_channelsrt   Úvit_stemÚ	conv_initÚ
isinstanceÚlistÚtupleZ
stage_num1Z
stage_num2Z
stage_num3ÚsumÚuse_pos_embedÚgrad_checkpointingrP   ZlinspaceÚstemr   Úpatch_embed1r%   Z
Sequentialr&   ÚBatchNorm2dZReLUÚ	ParameterZzerosÚ
pos_embed1r)   Úpos_dropÚrangeÚstage1Úpatch_embed2Ú
pos_embed2Ústage2Úpatch_embed3Ú
pos_embed3Ústage3Únum_featuresÚnormr
   Úglobal_poolÚ	head_dropÚheadr   ÚapplyÚ_init_weights)r/   rt   ru   rv   rŒ   r�   rw   ÚdepthrC   rf   Z	drop_rateZpos_drop_rater†   rƒ   Zdrop_path_raterg   r„   r”   r"   rŽ   r$   r¦   r�   Ú
embed_normr¨   r2   )rƒ   r„   r…   rw   r$   rf   rg   rC   ru   r†   r"   r5   r   ¢   sØ    

úú

ýú ò
ú ò
ú  ò
zVisformer.__init__c                 C   sŠ   t |tjƒr6t|jdd� |jd k	r†tj |jd¡ nPt |tjƒr†| j	r^tjj
|jddd� nt|jdd� |jd k	r†tj |jd¡ d S )Nr‰   rŠ   r   Zfan_outZrelu)ÚmodeZnonlinearityr   )r�   r%   ZLinearr   Úweightr   ÚinitZ	constant_r&   r�   Zkaiming_normal_)r/   Úmr4   r4   r5   rª   f  s    

zVisformer._init_weightsc                 C   s   t d|rdndd fddgd�S )Nz^patch_embed1|pos_embed1|stemz^stage(\d+)\.(\d+))z^(?:patch_embed|pos_embed)(\d+))r   )z^norm)iŸ† )r–   Úblocks)Údict)r/   Zcoarser4   r4   r5   Úgroup_matchers  s    ýþzVisformer.group_matcherc                 C   s
   || _ d S r6   )r•   )r/   Úenabler4   r4   r5   Úset_grad_checkpointing~  s    z Visformer.set_grad_checkpointingc                 C   s   | j S r6   )r¨   )r/   r4   r4   r5   Úget_classifier‚  s    zVisformer.get_classifierc                 C   s$   || _ t| j| j |d�\| _| _d S )Nrˆ   )rŒ   r
   r¤   r¦   r¨   )r/   rŒ   r¦   r4   r4   r5   Úreset_classifier†  s    zVisformer.reset_classifierc                 C   s  | j d k	r|   |¡}|  |¡}| jd k	r8|  || j ¡}| jrVtj ¡ sVt| j	|ƒ}n
|  	|¡}| j
d k	rŽ|  
|¡}| jd k	rŽ|  || j ¡}| jr¬tj ¡ s¬t| j|ƒ}n
|  |¡}| jd k	rä|  |¡}| jd k	rä|  || j ¡}| j�rtj ¡ �st| j|ƒ}n
|  |¡}|  |¡}|S r6   )r–   r—   rš   r›   r•   rP   r[   Zis_scriptingr   r�   rž   rŸ   r    r¡   r¢   r£   r¥   r7   r4   r4   r5   Úforward_featuresŠ  s0    













zVisformer.forward_features)Ú
pre_logitsc                 C   s&   |   |¡}|  |¡}|r|S |  |¡S r6   )r¦   r§   r¨   )r/   r8   r¹   r4   r4   r5   Úforward_head®  s    

zVisformer.forward_headc                 C   s   |   |¡}|  |¡}|S r6   )r¸   rº   r7   r4   r4   r5   r9   ³  s    

zVisformer.forward)F)T)ro   )F)r:   r;   r<   r	   r   rª   rP   r[   Úignorer³   rµ   r¶   r·   r¸   r]   rº   r9   r>   r4   r4   r2   r5   r   ¡   sH   è E


$Fc                 K   s(   |  dd ¡rtdƒ‚tt| |f|Ž}|S )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.)ÚgetÚRuntimeErrorr   r   )ÚvariantÚ
pretrainedZdefault_cfgÚkwargsÚmodelr4   r4   r5   Ú_create_visformer¹  s    rÂ   Ú c                 K   s    | ddddddt tddd	œ|–S )
Nrk   )r   ri   ri   )r{   r{   gÍÌÌÌÌÌì?ZbicubicTzstem.0r¨   )ÚurlrŒ   Z
input_sizeZ	pool_sizeZcrop_pctÚinterpolationZfixed_input_sizeZmeanr‹   Z
first_convÚ
classifierr   )rÄ   rÀ   r4   r4   r5   Ú_cfgÀ  s          ûúrÇ   ztimm/)Z	hf_hub_id)zvisformer_tiny.in1kzvisformer_small.in1k)Úreturnc                 K   s@   t ddddddddtjd	tjd
�}tdd| it |f|Ž—Ž}|S )Nrj   éÀ   ©r{   rM   rM   r   r`   r   Ú011Ú100T©r�   rw   r«   rC   rf   r$   r„   r"   rg   r�   r¬   Úvisformer_tinyr¿   )rÎ   ©r²   r%   r˜   rÂ   ©r¿   rÀ   Z	model_cfgrÁ   r4   r4   r5   rÎ   Ñ  s            ýrÎ   c                 K   s@   t ddddddddtjd	tjd
�}tdd| it |f|Ž—Ž}|S )Nrl   rm   rÊ   r   r`   r   rË   rÌ   TrÍ   Úvisformer_smallr¿   )rÑ   rÏ   rÐ   r4   r4   r5   rÑ   Û  s            ýrÑ   )FN)rÃ   )F)F)!Ú__doc__rP   Ztorch.nnr%   Z	timm.datar   r   Ztimm.layersr   r   r   r   r	   r
   r   Z_builderr   Z_manipulater   Ú	_registryr   r   Ú__all__ÚModuler   r?   r_   r   rÂ   rÇ   Zdefault_cfgsrÎ   rÑ   r4   r4   r4   r5   Ú<module>   s0   	$3'1  

þ	