U
    ¯Ê¹j†C  ã                
   @   s  d Z ddlZddlmZmZmZmZ ddlZddlm	Z	 ddl
m	  mZ ddlm  mZ ddlmZmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZmZ d	gZG d
d„ de	j ƒZ!G dd„ de	j ƒZ"G dd„ de	j ƒZ#G dd„ de	j ƒZ$G dd„ de	j ƒZ%G dd	„ d	e	j ƒZ&dd„ Z'd/dd„Z(d0dd„Z)ee)dd�e)dd�e)dd�e)dd�e)dd�e)dd�e)dd�dœƒZ*ed1e&d œd!d"„ƒZ+ed2e&d œd#d$„ƒZ,ed3e&d œd%d&„ƒZ-ed4e&d œd'd(„ƒZ.ed5e&d œd)d*„ƒZ/ed6e&d œd+d,„ƒZ0ed7e&d œd-d.„ƒZ1dS )8a   Pyramid Vision Transformer v2

@misc{wang2021pvtv2,
      title={PVTv2: Improved Baselines with Pyramid Vision Transformer},
      author={Wenhai Wang and Enze Xie and Xiang Li and Deng-Ping Fan and Kaitao Song and Ding Liang and
        Tong Lu and Ping Luo and Ling Shao},
      year={2021},
      eprint={2106.13797},
      archivePrefix={arXiv},
      primaryClass={cs.CV}
}

Based on Apache 2.0 licensed code at https://github.com/whai362/PVT

Modifications and timm support by / Copyright 2022, Ross Wightman
é    N)ÚTupleÚListÚCallableÚUnion©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚ	to_2tupleÚ	to_ntupleÚtrunc_normal_Ú	LayerNormÚuse_fused_attné   )Úbuild_model_with_cfg)Úregister_modelÚgenerate_default_cfgsÚPyramidVisionTransformerV2c                       s<   e Zd Zddejddf‡ fdd„	Zee dœdd„Z‡  Z	S )	ÚMlpWithDepthwiseConvNç        Fc              	      s~   t ƒ  ¡  |p|}|p|}t ||¡| _|r4t ¡ nt ¡ | _tj||dddd|d�| _	|ƒ | _
t ||¡| _t |¡| _d S )Né   r   T)ÚbiasÚgroups)ÚsuperÚ__init__ÚnnÚLinearÚfc1ZReLUÚIdentityÚreluÚConv2dÚdwconvÚactÚfc2ÚDropoutÚdrop)ÚselfÚin_featuresÚhidden_featuresZout_featuresÚ	act_layerr%   Ú
extra_relu©Ú	__class__© úQ/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/pvt_v2.pyr   #   s    	
zMlpWithDepthwiseConv.__init__©Ú	feat_sizec                 C   sˆ   |   |¡}|j\}}}| dd¡ |||d |d ¡}|  |¡}|  |¡}| d¡ dd¡}|  |¡}|  |¡}|  	|¡}|  |¡}|S )Nr   é   r   )
r   ÚshapeÚ	transposeÚviewr   r!   Úflattenr"   r%   r#   )r&   Úxr0   ÚBÚNÚCr-   r-   r.   Úforward6   s    
 





zMlpWithDepthwiseConv.forward)
Ú__name__Ú
__module__Ú__qualname__r   ÚGELUr   r   Úintr:   Ú__classcell__r-   r-   r+   r.   r   "   s   ùr   c                       sB   e Zd ZU ejje ed< d‡ fdd„	Ze	e
 d	œd
d„Z‡  ZS )Ú	AttentionÚ
fused_attné   r   FTr   c                    s&  t ƒ  ¡  || dks,td|› d|› d�ƒ‚|| _|| _|| | _| jd | _tƒ | _t	j
|||d�| _t	j
||d |d�| _t	 |¡| _t	 
||¡| _t	 |¡| _|sìd | _|dkrØt	j||||d	�| _t	 |¡| _nd | _d | _d | _n6t	 d
¡| _t	j||ddd	�| _t	 |¡| _t	 ¡ | _d S )Nr   zdim z  should be divided by num_heads Ú.g      à¿)r   r1   r   )Úkernel_sizeÚstrideé   )r   r   ÚAssertionErrorÚdimÚ	num_headsÚhead_dimÚscaler   rB   r   r   ÚqÚkvr$   Ú	attn_dropÚprojÚ	proj_dropÚpoolr    Úsrr   Únormr"   ZAdaptiveAvgPool2dr>   )r&   rI   rJ   Úsr_ratioÚlinear_attnÚqkv_biasrO   rQ   r+   r-   r.   r   G   s0    

"
zAttention.__init__r/   c                 C   s  |j \}}}|\}}|  |¡ ||| jd¡ dddd¡}| jd k	rÀ| ddd¡ ||||¡}|  |  |¡¡ ||d¡ ddd¡}|  |¡}|  |¡}|  	|¡ |dd| j| j
¡ ddddd¡}	n¤| jd k	�r:| ddd¡ ||||¡}|  |¡ ||d¡ ddd¡}|  |¡}|  	|¡ |dd| j| j
¡ ddddd¡}	n*|  	|¡ |dd| j| j
¡ ddddd¡}	|	 d¡\}
}| j�ržtj||
|| j�r”| jjndd�}n8|| j }||
 d	d¡ }|jdd
�}|  |¡}|| }| dd¡ |||¡}|  |¡}|  |¡}|S )Néÿÿÿÿr   r1   r   r   é   r   )Z	dropout_péþÿÿÿ©rI   )r2   rM   ÚreshaperJ   ÚpermuterR   rS   rT   r"   rN   rK   ZunbindrB   ÚFZscaled_dot_product_attentionZtrainingrO   ÚprL   r3   ZsoftmaxrP   rQ   )r&   r6   r0   r7   r8   r9   ÚHÚWrM   rN   ÚkÚvÚattnr-   r-   r.   r:   o   s6    $
$

,
,*$



zAttention.forward)rC   r   FTr   r   )r;   r<   r=   ÚtorchÚjitÚFinalÚboolÚ__annotations__r   r   r?   r:   r@   r-   r-   r+   r.   rA   D   s   
      ø(rA   c                	       sD   e Zd Zdddddddejef	‡ fdd„	Zee dœdd	„Z	‡  Z
S )
ÚBlockç      @r   Fr   c              	      sŠ   t ƒ  ¡  ||ƒ| _t|||||||d�| _|	dkr<t|	ƒnt ¡ | _||ƒ| _	t
|t|| ƒ|
||d�| _|	dkr|t|	ƒnt ¡ | _d S )N)rJ   rU   rV   rW   rO   rQ   r   )r'   r(   r)   r%   r*   )r   r   Únorm1rA   rd   r	   r   r   Ú
drop_path1Únorm2r   r?   ÚmlpÚ
drop_path2)r&   rI   rJ   Ú	mlp_ratiorU   rV   rW   rQ   rO   Ú	drop_pathr)   Ú
norm_layerr+   r-   r.   r   •   s*    

ù	

ûzBlock.__init__r/   c                 C   s<   ||   |  |  |¡|¡¡ }||  |  |  |¡|¡¡ }|S ©N)rm   rd   rl   rp   ro   rn   )r&   r6   r0   r-   r-   r.   r:   º   s    zBlock.forward)r;   r<   r=   r   r>   r   r   r   r?   r:   r@   r-   r-   r+   r.   rj   “   s   ô%rj   c                       s*   e Zd ZdZd
‡ fdd„	Zdd	„ Z‡  ZS )ÚOverlapPatchEmbedz Image to Patch Embedding
    rG   rY   r   é   c                    sf   t ƒ  ¡  t|ƒ}t|ƒ|ks&tdƒ‚|| _tj|||||d d |d d fd�| _t 	|¡| _
d S )Nz!Set larger patch_size than strider   r1   r   )rF   Úpadding)r   r   r
   ÚmaxrH   Ú
patch_sizer   r    rP   r   rT   )r&   ry   rF   Úin_chansÚ	embed_dimr+   r-   r.   r   Ä   s    
   þzOverlapPatchEmbed.__init__c                 C   s(   |   |¡}| dddd¡}|  |¡}|S )Nr   r1   r   r   )rP   r]   rT   ©r&   r6   r-   r-   r.   r:   Î   s    

zOverlapPatchEmbed.forward)rG   rY   r   rv   )r;   r<   r=   Ú__doc__r   r:   r@   r-   r-   r+   r.   ru   Á   s   
ru   c                       sd   e Zd Zdddddddddef
eeeeeeeeeeeeee ef e	dœ‡ fdd	„Z
d
d„ Z‡  ZS )ÚPyramidVisionTransformerStageTrC   r   Frk   r   )rI   Údim_outÚdepthÚ
downsamplerJ   rU   rV   rq   rW   rQ   rO   rr   rs   c                    sx   t ƒ  ¡  d| _|r(tdd|ˆd�| _n|ˆks4t‚d | _t ‡ ‡‡‡‡‡‡‡‡‡	f
dd„t|ƒD ƒ¡| _	ˆˆƒ| _
d S )NFr   r1   ©ry   rF   rz   r{   c                    s:   g | ]2}t ˆˆˆ	ˆˆˆˆˆ tˆtƒr,ˆ| nˆˆd �
‘qS ))
rI   rJ   rU   rV   rq   rW   rQ   rO   rr   rs   )rj   Ú
isinstanceÚlist)Ú.0Úi©
rO   r   rr   rV   rq   rs   rJ   rQ   rW   rU   r-   r.   Ú
<listcomp>ô   s   õöz:PyramidVisionTransformerStage.__init__.<locals>.<listcomp>)r   r   Úgrad_checkpointingru   r�   rH   r   Z
ModuleListÚrangeÚblocksrT   )r&   rI   r   r€   r�   rJ   rU   rV   rq   rW   rQ   rO   rr   rs   r+   r‡   r.   r   Ö   s    
ü
 õ
z&PyramidVisionTransformerStage.__init__c                 C   s¤   | j d k	r|   |¡}|j\}}}}||f}| |d|¡}| jD ].}| jrbtj ¡ sbt |||¡}q>|||ƒ}q>|  	|¡}| ||d |d d¡ 
dddd¡ ¡ }|S )NrX   r   r   r   r1   )r�   r2   r\   r‹   r‰   re   rf   Zis_scriptingÚ
checkpointrT   r]   Ú
contiguous)r&   r6   r7   r`   ra   r9   r0   Zblkr-   r-   r.   r:     s    



(z%PyramidVisionTransformerStage.forward)r;   r<   r=   r   r?   rh   Úfloatr   r   r   r   r:   r@   r-   r-   r+   r.   r~   Õ   s4   òò-r~   c                       s²   e Zd Zddddddddd	d
ddddef‡ fdd„	Zdd„ Zdd„ Zejj	dd„ ƒZ
ejj	d$dd„ƒZejj	d%dd„ƒZdd„ Zd&dd„Zdd„ Zd'edœd d!„Zd"d#„ Z‡  ZS )(r   r   éè  Úavg©r   rY   é   r   )é@   é€   é   é   )r   r1   rY   rC   )rC   rY   r1   r   )ç       @r—   rk   rk   TFr   c                    s–  t ƒ  ¡  || _|dkst‚|| _|| _t|ƒ}t|ƒ|ƒ}t|ƒ|ƒ}t|ƒ|ƒ}t|ƒ|ksdt‚g | _t	dd||d d�| _
dd„ t d|t|ƒ¡ |¡D ƒ}d}|d }g }t|ƒD ]„}|t||| || |dk|| || || |
|	|||| |d�g7 }|| }||| 7 }|  jt|dd	|  d
|› �d�g7  _qºtj|Ž | _|d | _t |¡| _|dk�r|t |d |¡nt ¡ | _|  | j¡ d S )N©r�   Ú rG   rY   r   r‚   c                 S   s   g | ]}|  ¡ ‘qS r-   )Útolist)r…   r6   r-   r-   r.   rˆ   ;  s     z7PyramidVisionTransformerV2.__init__.<locals>.<listcomp>)rI   r   r€   r�   rJ   rU   rq   rV   rW   rQ   rO   rr   rs   r1   ústages.)Znum_chsZ	reductionÚmodulerX   )r   r   Únum_classesrH   Úglobal_poolÚdepthsÚlenr   Zfeature_inforu   Úpatch_embedre   ZlinspaceÚsumÚsplitrŠ   r~   Údictr   Z
SequentialÚstagesZnum_featuresr$   Ú	head_dropr   r   ÚheadÚapplyÚ_init_weights)r&   rz   r�   rž   rŸ   Ú
embed_dimsrJ   Z	sr_ratiosÚ
mlp_ratiosrW   ÚlinearZ	drop_rateZproj_drop_rateZattn_drop_rateZdrop_path_raters   Z
num_stagesZdprÚcurZprev_dimr¥   r†   r+   r-   r.   r     sZ    
ü"ó
*
$z#PyramidVisionTransformerV2.__init__c                 C   s¦   t |tjƒrBt|jdd� t |tjƒr¢|jd k	r¢tj |jd¡ n`t |tjƒr¢|j	d |j	d  |j
 }||j }|jj dt d| ¡¡ |jd k	r¢|jj ¡  d S )Ng{®Gáz”?)Ústdr   r   g       @)rƒ   r   r   r   Úweightr   ÚinitZ	constant_r    rE   Zout_channelsr   ÚdataZnormal_ÚmathÚsqrtZzero_)r&   ÚmZfan_outr-   r-   r.   r©   [  s    

z(PyramidVisionTransformerV2._init_weightsc                 C   s   d| j _d S )NF)r¡   Zrequires_grad©r&   r-   r-   r.   Úfreeze_patch_embg  s    z+PyramidVisionTransformerV2.freeze_patch_embc                 C   s   i S rt   r-   rµ   r-   r-   r.   Úno_weight_decayj  s    z*PyramidVisionTransformerV2.no_weight_decayc                 C   s   t ddd�}|S )Nz^patch_embedz^stages\.(\d+))Ústemr‹   )r¤   )r&   ZcoarseZmatcherr-   r-   r.   Úgroup_matchern  s
    þz(PyramidVisionTransformerV2.group_matcherc                 C   s   | j D ]
}||_qd S rt   )r¥   r‰   )r&   ÚenableÚsr-   r-   r.   Úset_grad_checkpointingv  s    
z1PyramidVisionTransformerV2.set_grad_checkpointingc                 C   s   | j S rt   )r§   rµ   r-   r-   r.   Úget_classifier{  s    z)PyramidVisionTransformerV2.get_classifierNc                 C   sD   || _ |d k	r |dkst‚|| _|dkr6t | j|¡nt ¡ | _d S )Nr˜   r   )r�   rH   rž   r   r   r{   r   r§   )r&   r�   rž   r-   r-   r.   Úreset_classifier~  s
    z+PyramidVisionTransformerV2.reset_classifierc                 C   s   |   |¡}|  |¡}|S rt   )r¡   r¥   r|   r-   r-   r.   Úforward_features…  s    

z+PyramidVisionTransformerV2.forward_features)Ú
pre_logitsc                 C   s.   | j r|jdd�}|  |¡}|r$|S |  |¡S )N)rX   rZ   r[   )rž   Úmeanr¦   r§   )r&   r6   rÀ   r-   r-   r.   Úforward_headŠ  s    
z'PyramidVisionTransformerV2.forward_headc                 C   s   |   |¡}|  |¡}|S rt   )r¿   rÂ   r|   r-   r-   r.   r:   �  s    

z"PyramidVisionTransformerV2.forward)F)T)N)F)r;   r<   r=   r   r   r©   r¶   re   rf   Úignorer·   r¹   r¼   r½   r¾   r¿   rh   rÂ   r:   r@   r-   r-   r+   r.   r     s:   ðE

c                 C   s    d| kr| S i }ddl }|  ¡ D ]z\}}| d¡rb| dd¡}| dd¡}| dd	¡}| d
d¡}| dd¡}| ddd„ |¡}| ddd„ |¡}|||< q |S )z$ Remap original checkpoints -> timm zpatch_embed.proj.weightr   Nr¡   Zpatch_embed1Zpatch_embed2zstages.1.downsampleZpatch_embed3zstages.2.downsampleZpatch_embed4zstages.3.downsamplezdwconv.dwconvr!   zblock(\d+).(\d+)c                 S   s$   dt |  d¡ƒd › d|  d¡› �S )Nr›   r   z.blocks.r1   ©r?   Úgroup©r6   r-   r-   r.   Ú<lambda>¤  ó    z'_checkpoint_filter_fn.<locals>.<lambda>z
^norm(\d+)c                 S   s   dt |  d¡ƒd › d�S )Nr›   r   z.normrÄ   rÆ   r-   r-   r.   rÇ   ¥  rÈ   )ÚreÚitemsÚ
startswithÚreplaceÚsub)Z
state_dictÚmodelZout_dictrÉ   rb   rc   r-   r-   r.   Ú_checkpoint_filter_fn–  s    

rÏ   Fc                 K   s>   t tdƒƒ}| d|¡}tt| |fttd|d�dœ|—Ž}|S )NrY   Úout_indicesT)Zflatten_sequentialrÐ   )Zpretrained_filter_fnZfeature_cfg)ÚtuplerŠ   Úpopr   r   rÏ   r¤   )ÚvariantÚ
pretrainedÚkwargsZdefault_out_indicesrÐ   rÎ   r-   r-   r.   Ú_create_pvt2ª  s    ý
ûúrÖ   r™   c                 K   s    | dddddt tdddd	œ|–S )
Nr�   )r   éà   r×   )rG   rG   gÍÌÌÌÌÌì?Zbicubiczpatch_embed.projr§   F)Úurlr�   Z
input_sizeZ	pool_sizeZcrop_pctÚinterpolationrÁ   r®   Z
first_convÚ
classifierZfixed_input_sizer   )rØ   rÕ   r-   r-   r.   Ú_cfg¸  s           üûrÛ   ztimm/)Z	hf_hub_id)zpvt_v2_b0.in1kzpvt_v2_b1.in1kzpvt_v2_b2.in1kzpvt_v2_b3.in1kzpvt_v2_b4.in1kzpvt_v2_b5.in1kzpvt_v2_b2_li.in1k)Úreturnc                 K   s(   t dddd�}tdd| it |f|Ž—ŽS )N©r1   r1   r1   r1   )é    r“   é    r•   ©r   r1   é   rC   ©rŸ   rª   rJ   Ú	pvt_v2_b0rÔ   )rã   ©r¤   rÖ   ©rÔ   rÕ   Z
model_argsr-   r-   r.   rã   Í  s    rã   c                 K   s(   t dddd�}tdd| it |f|Ž—ŽS )NrÝ   ©r“   r”   i@  r–   rà   râ   Ú	pvt_v2_b1rÔ   )rç   rä   rå   r-   r-   r.   rç   Ó  s    rç   c                 K   s(   t dddd�}tdd| it |f|Ž—ŽS )Nr‘   ræ   rà   râ   Ú	pvt_v2_b2rÔ   )rè   rä   rå   r-   r-   r.   rè   Ù  s    rè   c                 K   s(   t dddd�}tdd| it |f|Ž—ŽS )N)r   rY   é   r   ræ   rà   râ   Ú	pvt_v2_b3rÔ   )rê   rä   rå   r-   r-   r.   rê   ß  s    rê   c                 K   s(   t dddd�}tdd| it |f|Ž—ŽS )N)r   rC   é   r   ræ   rà   râ   Ú	pvt_v2_b4rÔ   )rì   rä   rå   r-   r-   r.   rì   å  s    rì   c                 K   s*   t ddddd�}tdd| it |f|Ž—ŽS )	N)r   r’   é(   r   ræ   rà   )rY   rY   rY   rY   )rŸ   rª   rJ   r«   Ú	pvt_v2_b5rÔ   )rî   rä   rå   r-   r-   r.   rî   ë  s       ÿrî   c                 K   s*   t ddddd�}tdd| it |f|Ž—ŽS )	Nr‘   ræ   rà   T)rŸ   rª   rJ   r¬   Úpvt_v2_b2_lirÔ   )rï   rä   rå   r-   r-   r.   rï   ò  s       ÿrï   )F)r™   )F)F)F)F)F)F)F)2r}   r²   Útypingr   r   r   r   re   Ztorch.nnr   Ztorch.nn.functionalZ
functionalr^   Ztorch.utils.checkpointÚutilsrŒ   Z	timm.datar   r   Ztimm.layersr	   r
   r   r   r   r   Z_builderr   Ú	_registryr   r   Ú__all__ÚModuler   rA   rj   ru   r~   r   rÏ   rÖ   rÛ   Zdefault_cfgsrã   rç   rè   rê   rì   rî   rï   r-   r-   r-   r.   Ú<module>   sX    "O.@ 


ù