U
    ¯Ê¹j24  ã                   @   s.  d Z ddlZddlZddlmZ ddlmZ ddlmZm	Z	 ddl
mZmZmZmZmZ ddlmZ ddlmZ dd	lmZ d
gZd$dd„Zedddd�eddd�dœZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd
„ d
ejƒZdd„ Zd%dd„Zed&edœd d!„ƒZ ed'edœd"d#„ƒZ!dS )(a   Transformer in Transformer (TNT) in PyTorch

A PyTorch implement of TNT as described in
'Transformer in Transformer' - https://arxiv.org/abs/2103.00112

The official mindspore code is released and available at
https://gitee.com/mindspore/mindspore/tree/master/model_zoo/research/cv/TNT
é    N)Ú
checkpoint©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚMlpÚDropPathÚtrunc_normal_Ú_assertÚ	to_2tupleé   )Úbuild_model_with_cfg)Úregister_model)Úresize_pos_embedÚTNTÚ c                 K   s    | ddd dddt tdddœ|–S )	Néè  )é   éà   r   gÍÌÌÌÌÌì?ZbicubicTzpixel_embed.projÚhead)ÚurlÚnum_classesZ
input_sizeZ	pool_sizeZcrop_pctÚinterpolationZfixed_input_sizeÚmeanÚstdZ
first_convÚ
classifierr   )r   Úkwargs© r   úN/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/tnt.pyÚ_cfg   s          ûúr   zchttps://github.com/contrastive/pytorch-image-models/releases/download/TNT/tnt_s_patch16_224.pth.tar)ç      à?r   r   )r   r   r   )r   r   )Útnt_s_patch16_224Útnt_b_patch16_224c                       s*   e Zd ZdZd	‡ fdd„	Zdd„ Z‡  ZS )
Ú	Attentionz Multi-Head Attention
    é   Fç        c                    sˆ   t ƒ  ¡  || _|| _|| }|| _|d | _tj||d |d�| _tj|||d�| _	tj
|dd�| _t ||¡| _tj
|dd�| _d S )Ng      à¿é   ©ÚbiasT)Zinplace)ÚsuperÚ__init__Ú
hidden_dimÚ	num_headsÚhead_dimÚscaleÚnnÚLinearÚqkÚvÚDropoutÚ	attn_dropÚprojÚ	proj_drop)ÚselfÚdimr*   r+   Úqkv_biasr3   r5   r,   ©Ú	__class__r   r   r)   1   s    

zAttention.__init__c           
      C   sÆ   |j \}}}|  |¡ ||d| j| j¡ ddddd¡}| d¡\}}|  |¡ ||| jd¡ dddd¡}|| dd¡ | j	 }	|	j
dd�}	|  |	¡}	|	|  dd¡ ||d¡}|  |¡}|  |¡}|S )	Nr%   r   r   r   é   éÿÿÿÿéþÿÿÿ©r7   )Úshaper0   Úreshaper+   r,   ZpermuteZunbindr1   Ú	transposer-   Zsoftmaxr3   r4   r5   )
r6   ÚxÚBÚNÚCr0   ÚqÚkr1   Zattnr   r   r   Úforward?   s    *$


zAttention.forward)r#   Fr$   r$   ©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r)   rH   Ú__classcell__r   r   r9   r   r"   .   s   r"   c                	       s@   e Zd ZdZdddddddejejf	‡ fdd„	Zd	d
„ Z‡  Z	S )ÚBlockz TNT Block
    r;   é   ç      @Fr$   c                    sÔ   t ƒ  ¡  ||ƒ| _t|||||	|d�| _||ƒ| _t|t|d ƒ|||d�| _||ƒ| _	t
j|| |dd�| _||ƒ| _t|||||	|d�| _|
dkr t|
ƒnt
 ¡ | _||ƒ| _t|t|| ƒ|||d�| _d S )N)r+   r8   r3   r5   r;   )Zin_featuresZhidden_featuresZout_featuresÚ	act_layerZdropTr&   r$   )r(   r)   Únorm_inr"   Úattn_inÚnorm_mlp_inr   ÚintÚmlp_inÚ
norm1_projr.   r/   r4   Únorm_outÚattn_outr   ÚIdentityÚ	drop_pathÚnorm_mlpÚmlp)r6   r7   Údim_outÚ	num_pixelÚnum_heads_inÚnum_heads_outÚ	mlp_ratior8   r5   r3   r\   rR   Ú
norm_layerr9   r   r   r)   R   sL    

ú	

û

ú

ûzBlock.__init__c              
   C   sÎ   ||   |  |  |¡¡¡ }||   |  |  |¡¡¡ }| ¡ \}}}tj|d d …dd…f |d d …dd …f |  |  	|¡ 
||d d¡¡ gdd�}||   |  |  |¡¡¡ }||   |  |  |¡¡¡ }||fS )Nr   r   r<   r>   )r\   rT   rS   rW   rU   ÚsizeÚtorchÚcatr4   rX   r@   rZ   rY   r^   r]   )r6   Úpixel_embedÚpatch_embedrC   rD   rE   r   r   r   rH   Ž   s    DþzBlock.forward)
rJ   rK   rL   rM   r.   ZGELUÚ	LayerNormr)   rH   rN   r   r   r9   r   rO   O   s   ó<rO   c                       s*   e Zd ZdZd‡ fdd„	Zd	d
„ Z‡  ZS )Ú
PixelEmbedz Image to Pixel Embedding
    r   é   r   é0   r;   c                    s¦   t ƒ  ¡  t|ƒ}t|ƒ}|d |d  |d |d  f| _| jd | jd  }|| _|| _|| _‡ fdd„|D ƒ}|| _tj	|| jddˆ d�| _
tj||d�| _d S )	Nr   r   c                    s   g | ]}t  |ˆ  ¡‘qS r   )ÚmathÚceil)Ú.0Zps©Ústrider   r   Ú
<listcomp>©   s     z'PixelEmbed.__init__.<locals>.<listcomp>é   r   )Úkernel_sizeÚpaddingrr   )ru   rr   )r(   r)   r
   Ú	grid_sizeÚimg_sizeÚnum_patchesÚin_dimÚnew_patch_sizer.   ZConv2dr4   ZUnfoldÚunfold)r6   rx   Ú
patch_sizeÚin_chansrz   rr   ry   r{   r9   rq   r   r)   Ÿ   s    
"zPixelEmbed.__init__c                 C   sì   |j \}}}}t|| jd kd|› d|› d| jd › d| jd › d�	ƒ t|| jd kd|› d|› d| jd › d| jd › d�	ƒ |  |¡}|  |¡}| dd¡ || j | j| j	d | j	d ¡}|| }| || j | jd¡ dd¡}|S )	Nr   zInput image size (Ú*z) doesn't match model (r   z).r%   r<   )
r?   r	   rx   r4   r|   rA   r@   ry   rz   r{   )r6   rB   Ú	pixel_posrC   rE   ÚHÚWr   r   r   rH   ¯   s    (ÿ(ÿ

,zPixelEmbed.forward)r   rl   r   rm   r;   rI   r   r   r9   r   rk   œ   s   rk   c                       sÀ   e Zd ZdZdddddddd	d
d	dddddddejd
f‡ fdd„	Zdd„ Zej	j
dd„ ƒZej	j
d%dd„ƒZej	j
d&dd„ƒZej	j
dd„ ƒZd'dd„Zdd„ Zd(ed œd!d"„Zd#d$„ Z‡  ZS ))r   zC Transformer in Transformer - https://arxiv.org/abs/2103.00112
    r   rl   r   r   Útokeni   rm   rP   r;   rQ   Fr$   c                    sÆ  t ƒ  ¡  |dkst‚|| _|| _| | _| _d| _t|||||d�| _	| j	j
}|| _
| j	j}|d |d  }||| ƒ| _t || |¡| _||ƒ| _t t dd|¡¡| _t t d|d |¡¡| _t t d||d |d ¡¡| _tj|d�| _dd„ t d||¡D ƒ}g }t|ƒD ].}| t||||	|
|||||| |d	�¡ �qt |¡| _||ƒ| _t |¡| _ |dk�r‚t ||¡nt !¡ | _"t#| jd
d� t#| jd
d� t#| jd
d� |  $| j%¡ d S )N©r   rƒ   ÚavgF)rx   r}   r~   rz   rr   r   r   )Úpc                 S   s   g | ]}|  ¡ ‘qS r   )Úitem)rp   rB   r   r   r   rs   ò   s     z TNT.__init__.<locals>.<listcomp>)r7   r_   r`   ra   rb   rc   r8   r5   r3   r\   rd   ç{®Gáz”?©r   )&r(   r)   ÚAssertionErrorr   Úglobal_poolZnum_featuresÚ	embed_dimÚgrad_checkpointingrk   rh   ry   r{   rX   r.   r/   r4   Ú
norm2_projÚ	Parameterrf   ZzerosÚ	cls_tokenÚ	patch_posr€   r2   Úpos_dropZlinspaceÚrangeÚappendrO   Z
ModuleListÚblocksÚnormÚ	head_dropr[   r   r   ÚapplyÚ_init_weights)r6   rx   r}   r~   r   r‹   rŒ   Ú	inner_dimÚdepthZnum_heads_innerÚnum_heads_outerrc   r8   Z	drop_rateZpos_drop_rateZproj_drop_rateZattn_drop_rateZdrop_path_raterd   Zfirst_stridery   r{   r`   Zdprr•   Úir9   r   r   r)   À   s`    
û
 õ
 zTNT.__init__c                 C   sr   t |tjƒrBt|jdd� t |tjƒrn|jd k	rntj |jd¡ n,t |tjƒrntj |jd¡ tj |jd¡ d S )Nrˆ   r‰   r   g      ð?)	Ú
isinstancer.   r/   r   Úweightr'   ÚinitZ	constant_rj   )r6   Úmr   r   r   r™     s    zTNT._init_weightsc                 C   s
   dddhS )Nr‘   r€   r�   r   ©r6   r   r   r   Úno_weight_decay  s    zTNT.no_weight_decayc                 C   s   t dddgd�}|S )Nz=^cls_token|patch_pos|pixel_pos|pixel_embed|norm[12]_proj|proj)z^blocks\.(\d+)N)z^norm)iŸ† )Ústemr•   )Údict)r6   ZcoarseZmatcherr   r   r   Úgroup_matcher  s    þþzTNT.group_matcherTc                 C   s
   || _ d S ©N)r�   )r6   Úenabler   r   r   Úset_grad_checkpointing%  s    zTNT.set_grad_checkpointingc                 C   s   | j S r§   )r   r¢   r   r   r   Úget_classifier)  s    zTNT.get_classifierNc                 C   s>   || _ |d k	r|dkst‚|dkr0t | j|¡nt ¡ | _d S )Nr„   r   )r   rŠ   r.   r/   rŒ   r[   r   )r6   r   r‹   r   r   r   Úreset_classifier-  s    zTNT.reset_classifierc                 C   sÂ   |j d }|  || j¡}|  |  |  | || jd¡¡¡¡}tj	| j
 |dd¡|fdd�}|| j }|  |¡}| jrštj ¡ sš| jD ]}t|||ƒ\}}q‚n| jD ]}|||ƒ\}}q |  |¡}|S )Nr   r<   r   r>   )r?   rh   r€   rŽ   r4   rX   r@   ry   rf   rg   r�   Úexpandr‘   r’   r�   ÚjitZis_scriptingr•   r   r–   )r6   rB   rC   rh   ri   Zblkr   r   r   Úforward_features3  s    
"




zTNT.forward_features)Ú
pre_logitsc                 C   sX   | j r<| j dkr,|d d …dd …f jdd�n|d d …df }|  |¡}|rN|S |  |¡S )Nr…   r   r>   r   )r‹   r   r—   r   )r6   rB   r¯   r   r   r   Úforward_headF  s    6
zTNT.forward_headc                 C   s   |   |¡}|  |¡}|S r§   )r®   r°   )r6   rB   r   r   r   rH   L  s    

zTNT.forward)F)T)N)F)rJ   rK   rL   rM   r.   rj   r)   r™   rf   r­   Úignorer£   r¦   r©   rª   r«   r®   Úboolr°   rH   rN   r   r   r9   r   r   ½   sD   ìM	



c                 C   s:   | d j |jj kr6t| d |jt|ddƒ|jjƒ| d< | S )zJ convert patch embedding weight from manual patchify + linear proj to convr‘   Z
num_tokensr   )r?   r‘   r   Úgetattrrh   rw   )Z
state_dictÚmodelr   r   r   Úcheckpoint_filter_fnR  s     
 ÿrµ   Fc                 K   s0   |  dd ¡rtdƒ‚tt| |fdti|—Ž}|S )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.Zpretrained_filter_fn)ÚgetÚRuntimeErrorr   r   rµ   )ÚvariantÚ
pretrainedr   r´   r   r   r   Ú_create_tntZ  s      ÿþýrº   )Úreturnc                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )Nrl   i€  é   rP   é   F©r}   rŒ   rš   r›   rœ   r8   r    r¹   )r    ©r¥   rº   ©r¹   r   Z	model_cfgr´   r   r   r   r    e  s        þr    c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )Nrl   i€  é(   rP   é
   Fr¾   r!   r¹   )r!   r¿   rÀ   r   r   r   r!   n  s        þr!   )r   )F)F)F)"rM   rn   rf   Ztorch.nnr.   Ztorch.utils.checkpointr   Z	timm.datar   r   Ztimm.layersr   r   r   r	   r
   Z_builderr   Ú	_registryr   Zvision_transformerr   Ú__all__r   Zdefault_cfgsÚModuler"   rO   rk   r   rµ   rº   r    r!   r   r   r   r   Ú<module>   s@   
 þ ÿû!M! 
