U
    ¯Ê¹j^J  ã                	   @   sö  d Z ddlZddlmZ ddlmZ ddlZddlmZ ddl	m  m
Z ddlmZmZ ddlmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ dgZeeef Z eG dd„ dej!ƒƒZ"G dd„ dej!ƒZ#G dd„ dej!ƒZ$G dd„ dej!ƒZ%G dd„ dej!ƒZ&G dd„ dej!ƒZ'd.dd„Z(d/dd„Z)ee)dd�e)dd�e)dd�e)dd�e)dd�e)dd�d œƒZ*ed0e'd!œd"d#„ƒZ+ed1e'd!œd$d%„ƒZ,ed2e'd!œd&d'„ƒZ-ed3e'd!œd(d)„ƒZ.ed4e'd!œd*d+„ƒZ/ed5e'd!œd,d-„ƒZ0dS )6zô Twins
A PyTorch impl of : `Twins: Revisiting the Design of Spatial Attention in Vision Transformers`
    - https://arxiv.org/pdf/2104.13840.pdf

Code/weights from https://github.com/Meituan-AutoML/Twins, original copyright/license info below

é    N)Úpartial)ÚTuple©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚMlpÚDropPathÚ	to_2tupleÚtrunc_normal_Úuse_fused_attné   )Úbuild_model_with_cfg)Úregister_notrace_module)Úregister_modelÚgenerate_default_cfgs)Ú	AttentionÚTwinsc                       sB   e Zd ZU dZejje ed< d‡ fdd„	Z	e
dœd	d
„Z‡  ZS )ÚLocallyGroupedAttnz( LSA: self attention within a group
    Ú
fused_attné   ç        r   c                    s¨   |dkst ‚tt| ƒ ¡  || dks<t d|› d|› d�ƒ‚|| _|| _|| }|d | _tƒ | _t	j
||d dd	�| _t	 |¡| _t	 
||¡| _t	 |¡| _|| _d S )
Nr   r   údim ú  should be divided by num_heads Ú.ç      à¿é   T©Úbias)ÚAssertionErrorÚsuperr   Ú__init__ÚdimÚ	num_headsÚscaler   r   ÚnnÚLinearÚqkvÚDropoutÚ	attn_dropÚprojÚ	proj_dropÚws)Úselfr!   r"   r(   r*   r+   Úhead_dim©Ú	__class__© úP/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/twins.pyr    (   s    "
zLocallyGroupedAttn.__init__©Úsizec              	   C   s  |j \}}}|\}}| ||||¡}d }}	| j|| j  | j }
| j|| j  | j }t |dd||
|	|f¡}|j \}}}}|| j || j  }}| ||| j|| j|¡ dd¡}|  |¡ ||| | j| j d| j|| j ¡ 	dddddd¡}| 
d¡\}}}| j�r.tj|||| j�r$| jjndd�}n8|| j }|| d	d
¡ }|jd
d�}|  |¡}|| }| dd¡ |||| j| j|¡}| dd¡ ||| j || j |¡}|
dk�s¾|dk�râ|d d …d |…d |…d d …f  ¡ }| |||¡}|  |¡}|  |¡}|S )Nr   é   r   r   é   é   r   ©Z	dropout_péþÿÿÿéÿÿÿÿ©r!   )ÚshapeÚviewr+   ÚFÚpadÚreshapeÚ	transposer&   r"   ÚpermuteÚunbindr   Úscaled_dot_product_attentionÚtrainingr(   Úpr#   ÚsoftmaxÚ
contiguousr)   r*   )r,   Úxr3   ÚBÚNÚCÚHÚWZpad_lZpad_tZpad_rZpad_bÚ_ZHpZWpÚ_hÚ_wr&   ÚqÚkÚvÚattnr0   r0   r1   Úforward9   s\     
  
   ÿ     ÿ  þ

 $$

zLocallyGroupedAttn.forward)r   r   r   r   ©Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtorchÚjitÚFinalÚboolÚ__annotations__r    ÚSize_rU   Ú__classcell__r0   r0   r.   r1   r   "   s   
r   c                       sB   e Zd ZU dZejje ed< d‡ fdd„	Z	e
dœd	d
„Z‡  ZS )ÚGlobalSubSampleAttnzQ GSA: using a  key to summarize the information for a group to be efficient.
    r   r   r   r   c                    sà   t ƒ  ¡  || dks,td|› d|› d�ƒ‚|| _|| _|| }|d | _tƒ | _tj	||dd�| _
tj	||d dd�| _t |¡| _t 	||¡| _t |¡| _|| _|d	krÐtj||||d
�| _t |¡| _nd | _d | _d S )Nr   r   r   r   r   Tr   r4   r   ©Úkernel_sizeÚstride)r   r    r   r!   r"   r#   r   r   r$   r%   rQ   Úkvr'   r(   r)   r*   Úsr_ratioÚConv2dÚsrÚ	LayerNormÚnorm)r,   r!   r"   r(   r*   rg   r-   r.   r0   r1   r    ‰   s$    
"
zGlobalSubSampleAttn.__init__r2   c                 C   sR  |j \}}}|  |¡ ||| j|| j ¡ dddd¡}| jd k	r„| ddd¡j||f|žŽ }|  |¡ ||d¡ ddd¡}|  |¡}|  |¡ |dd| j|| j ¡ ddddd¡}| d¡\}}	| j	rìt
jjj|||	| jrâ| jjndd�}n8|| j }|| d	d¡ }
|
jdd
�}
|  |
¡}
|
|	 }| dd¡ |||¡}|  |¡}|  |¡}|S )Nr   r4   r   r   r9   r5   r   r7   r8   r:   )r;   rQ   r?   r"   rA   ri   rk   rf   rB   r   r[   r$   Ú
functionalrC   rD   r(   rE   r#   r@   rF   r)   r*   )r,   rH   r3   rI   rJ   rK   rQ   rf   rR   rS   rT   r0   r0   r1   rU   ¡   s0    *

.  þ



zGlobalSubSampleAttn.forward)r   r   r   r   rV   r0   r0   r.   r1   rb   „   s   
rb   c                       s@   e Zd Zddddejejddf‡ fdd„	Zedœdd	„Z‡  Z	S )
ÚBlockg      @r   r   Nc                    s¼   t ƒ  ¡  ||ƒ| _|
d kr2t||dd ||ƒ| _n.|
dkrNt|||||	ƒ| _nt|||||
ƒ| _|dkrpt|ƒnt 	¡ | _
||ƒ| _t|t|| ƒ||d�| _|dkr®t|ƒnt 	¡ | _d S )NFr   r   )Zin_featuresZhidden_featuresÚ	act_layerÚdrop)r   r    Únorm1r   rT   rb   r   r   r$   ÚIdentityÚ
drop_path1Únorm2r   ÚintÚmlpÚ
drop_path2)r,   r!   r"   Ú	mlp_ratior*   r(   Ú	drop_pathrn   Ú
norm_layerrg   r+   r.   r0   r1   r    Á   s     



üzBlock.__init__r2   c                 C   s:   ||   |  |  |¡|¡¡ }||  |  |  |¡¡¡ }|S ©N)rr   rT   rp   rv   ru   rs   )r,   rH   r3   r0   r0   r1   rU   á   s    zBlock.forward)
rW   rX   rY   r$   ZGELUrj   r    r`   rU   ra   r0   r0   r.   r1   rm   ¿   s   õ rm   c                       s4   e Zd Zd
‡ fdd„	Zedœdd„Zdd	„ Z‡  ZS )ÚPosConvé   r   c                    s8   t t| ƒ ¡  t tj||d|dd|d�¡| _|| _d S )Nr   r   T)r   Úgroups)r   r{   r    r$   Z
Sequentialrh   r)   re   )r,   Úin_chansÚ	embed_dimre   r.   r0   r1   r    é   s
    ÿzPosConv.__init__r2   c                 C   sX   |j \}}}| dd¡j||f|žŽ }|  |¡}| jdkrB||7 }| d¡ dd¡}|S )Nr   r4   )r;   r@   r<   r)   re   Úflatten)r,   rH   r3   rI   rJ   rK   Zcnn_feat_tokenr0   r0   r1   rU   ð   s    

zPosConv.forwardc                 C   s   dd„ t dƒD ƒS )Nc                 S   s   g | ]}d | ‘qS )zproj.%d.weightr0   ©Ú.0Úir0   r0   r1   Ú
<listcomp>ú   s     z+PosConv.no_weight_decay.<locals>.<listcomp>r5   )Úrange©r,   r0   r0   r1   Úno_weight_decayù   s    zPosConv.no_weight_decay)r|   r   )rW   rX   rY   r    r`   rU   r‡   ra   r0   r0   r.   r1   r{   ç   s   	r{   c                       s:   e Zd ZdZd‡ fdd„	Zeejef dœd	d
„Z	‡  Z
S )Ú
PatchEmbedz Image to Patch Embedding
    éà   é   r   r|   c                    s¼   t ƒ  ¡  t|ƒ}t|ƒ}|| _|| _|d |d  dkrN|d |d  dksdtd|› d|› d�ƒ‚|d |d  |d |d   | _| _| j| j | _t	j
||||d�| _t	 |¡| _d S )Nr   r   z	img_size z! should be divided by patch_size r   rc   )r   r    r	   Úimg_sizeÚ
patch_sizer   rL   rM   Znum_patchesr$   rh   r)   rj   rk   )r,   r‹   rŒ   r~   r   r.   r0   r1   r      s    
*ÿ&zPatchEmbed.__init__©Úreturnc                 C   sT   |j \}}}}|  |¡ d¡ dd¡}|  |¡}|| jd  || jd  f}||fS )Nr4   r   r   )r;   r)   r€   r@   rk   rŒ   )r,   rH   rI   rK   rL   rM   Zout_sizer0   r0   r1   rU     s
    
zPatchEmbed.forward)r‰   rŠ   r   r|   )rW   rX   rY   rZ   r    r   r[   ZTensorr`   rU   ra   r0   r0   r.   r1   rˆ   ý   s   rˆ   c                       sÆ   e Zd ZdZdddddddd	d
dddddddeejdd�ef‡ fdd„	Ze	j
jdd„ ƒZe	j
jd'dd„ƒZe	j
jd(dd„ƒZe	j
jdd„ ƒZd)dd„Zdd„ Zd d!„ Zd*ed"œd#d$„Zd%d&„ Z‡  ZS )+r   z™ Twins Vision Transfomer (Revisiting Spatial Attention)

    Adapted from PVT (PyramidVisionTransformer) class at https://github.com/whai362/PVT.git
    r‰   r5   r   éè  Úavg)é@   é€   é   é   )r   r4   r5   r   )r5   r5   r5   r5   )r   r5   é   r   )r   r5   r4   r   Nr   g�íµ ÷Æ°>)Úepsc                    sž  t ƒ  ¡  || _|| _|	| _ˆ| _ˆd | _d| _t|ƒ}|}t	 
¡ | _t	 
¡ | _tt|	ƒƒD ]T}| j t|ˆ	|ˆ| ƒ¡ | j t	j|d�¡ ˆ| }t‡	fdd„|D ƒƒ}d‰	q^t	 
¡ | _dd„ t d	|t|	ƒ¡D ƒ‰d	‰tt|	ƒƒD ]R‰t	 
‡ ‡‡‡‡‡‡‡‡‡
‡‡fd
d„t|	ˆ ƒD ƒ¡}| j |¡ ˆ|	ˆ 7 ‰qêt	 
dd„ ˆD ƒ¡| _ˆ| jƒ| _t	 |¡| _|d	k�r„t	 | j|¡nt	 ¡ | _|  | j¡ d S )Nr9   F)rE   c                 3   s   | ]}|ˆ  V  qd S rz   r0   )r‚   Út)rŒ   r0   r1   Ú	<genexpr>C  s     z!Twins.__init__.<locals>.<genexpr>r4   c                 S   s   g | ]}|  ¡ ‘qS r0   )Úitem)r‚   rH   r0   r0   r1   r„   G  s     z"Twins.__init__.<locals>.<listcomp>r   c                    sZ   g | ]R}ˆˆˆ ˆˆ ˆˆ ˆ	ˆ ˆˆ|  ˆˆ
ˆ ˆd ksF|d dkrJdnˆˆ d�	‘qS )Nr4   r   )	r!   r"   rw   r*   r(   rx   ry   rg   r+   r0   r�   )Úattn_drop_rateÚ	block_clsÚcurÚdprÚ
embed_dimsrR   Ú
mlp_ratiosry   r"   Úproj_drop_rateÚ	sr_ratiosÚwssr0   r1   r„   J  s   	÷
÷c                 S   s   g | ]}t ||ƒ‘qS r0   )r{   )r‚   r   r0   r0   r1   r„   X  s     )r   r    Únum_classesÚglobal_poolÚdepthsrž   Únum_featuresZgrad_checkpointingr	   r$   Z
ModuleListÚpatch_embedsÚ	pos_dropsr…   ÚlenÚappendrˆ   r'   ÚtupleÚblocksr[   ZlinspaceÚsumÚ	pos_blockrk   Ú	head_dropr%   rq   ÚheadÚapplyÚ_init_weights)r,   r‹   rŒ   r~   r£   r¤   rž   r"   rŸ   r¥   r¡   r¢   Z	drop_rateZpos_drop_rater    rš   Zdrop_path_ratery   r›   Zprev_chsrƒ   Ú_blockr.   )rš   r›   rœ   r�   rž   rR   rŸ   ry   r"   rŒ   r    r¡   r¢   r1   r      s>    




$	
÷"zTwins.__init__c                 C   s   t dd„ | j ¡ D ƒƒS )Nc                 S   s   g | ]\}}d | ‘qS )z
pos_block.r0   )r‚   ÚnrE   r0   r0   r1   r„   e  s     z)Twins.no_weight_decay.<locals>.<listcomp>)Úsetr®   Znamed_parametersr†   r0   r0   r1   r‡   c  s    zTwins.no_weight_decayFc                 C   s"   t d|rddgndddgd�}|S )Nz^patch_embeds.0)z)^(?:blocks|patch_embeds|pos_block)\.(\d+)N)z^norm)iŸ† )z^blocks\.(\d+)\.(\d+)N)z"^(?:patch_embeds|pos_block)\.(\d+))r   )Ústemr¬   )Údict)r,   ZcoarseZmatcherr0   r0   r1   Úgroup_matcherg  s    ýþýûzTwins.group_matcherTc                 C   s   |rt dƒ‚d S )Nz$gradient checkpointing not supported)r   )r,   Úenabler0   r0   r1   Úset_grad_checkpointingv  s    zTwins.set_grad_checkpointingc                 C   s   | j S rz   )r°   r†   r0   r0   r1   Úget_classifierz  s    zTwins.get_classifierc                 C   sD   || _ |d k	r |dkst‚|| _|dkr6t | j|¡nt ¡ | _d S )N)Ú r�   r   )r£   r   r¤   r$   r%   r¦   rq   r°   )r,   r£   r¤   r0   r0   r1   Úreset_classifier~  s
    zTwins.reset_classifierc                 C   sÔ   t |tjƒrBt|jdd� t |tjƒrÐ|jd k	rÐtj |jd¡ nŽt |tjƒrptj |jd¡ tj |jd¡ n`t |tj	ƒrÐ|j
d |j
d  |j }||j }|jj dt d| ¡¡ |jd k	rÐ|jj ¡  d S )Ng{®Gáz”?)Ústdr   g      ð?r   g       @)Ú
isinstancer$   r%   r
   Úweightr   ÚinitZ	constant_rj   rh   rd   Zout_channelsr}   ÚdataZnormal_ÚmathÚsqrtZzero_)r,   ÚmZfan_outr0   r0   r1   r²   …  s    

zTwins._init_weightsc                 C   sº   |j d }tt| j| j| j| jƒƒD ]†\}\}}}}||ƒ\}}||ƒ}t|ƒD ]$\}	}
|
||ƒ}|	dkrP|||ƒ}qP|t| jƒd k r$|j	|f|džŽ  
dddd¡ ¡ }q$|  |¡}|S )Nr   r   r9   r   r4   )r9   )r;   Ú	enumerateÚzipr§   r¨   r¬   r®   r©   r¥   r?   rA   rG   rk   )r,   rH   rI   rƒ   Zembedro   r¬   Zpos_blkr3   ÚjZblkr0   r0   r1   Úforward_features”  s    
ÿ
$
zTwins.forward_features)Ú
pre_logitsc                 C   s2   | j dkr|jdd�}|  |¡}|r(|S |  |¡S )Nr�   r   r:   )r¤   Úmeanr¯   r°   )r,   rH   rÊ   r0   r0   r1   Úforward_head£  s    

zTwins.forward_headc                 C   s   |   |¡}|  |¡}|S rz   )rÉ   rÌ   )r,   rH   r0   r0   r1   rU   ©  s    

zTwins.forward)F)T)N)F)rW   rX   rY   rZ   r   r$   rj   rm   r    r[   r\   Úignorer‡   r¸   rº   r»   r½   r²   rÉ   r^   rÌ   rU   ra   r0   r0   r.   r1   r     sB   íE


Fc                 K   s(   |  dd ¡rtdƒ‚tt| |f|Ž}|S )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.)ÚgetÚRuntimeErrorr   r   )ÚvariantÚ
pretrainedÚkwargsÚmodelr0   r0   r1   Ú_create_twins¯  s    rÔ   r¼   c                 K   s    | ddd dddt tdddœ|–S )	Nr�   )r   r‰   r‰   gÍÌÌÌÌÌì?ZbicubicTzpatch_embeds.0.projr°   )Úurlr£   Z
input_sizeZ	pool_sizeZcrop_pctÚinterpolationZfixed_input_sizerË   r¾   Z
first_convÚ
classifierr   )rÕ   rÒ   r0   r0   r1   Ú_cfg·  s          ûúrØ   ztimm/)Z	hf_hub_id)ztwins_pcpvt_small.in1kztwins_pcpvt_base.in1kztwins_pcpvt_large.in1kztwins_svt_small.in1kztwins_svt_base.in1kztwins_svt_large.in1kr�   c              
   K   sV   t dddddgdddd	gd	d	ddgd
ddd
gd	dddgd�}tdd| it |f|Ž—ŽS )Nr5   r‘   r’   é@  r”   r   r4   r6   r   r   r•   ©rŒ   rž   r"   rŸ   r¥   r¡   Útwins_pcpvt_smallrÑ   )rÛ   ©r·   rÔ   ©rÑ   rÒ   Z
model_argsr0   r0   r1   rÛ   Ì  s     
 
 

 
þrÛ   c              
   K   sV   t dddddgdddd	gd	d	ddgd
ddd
gd	dddgd�}tdd| it |f|Ž—ŽS )Nr5   r‘   r’   rÙ   r”   r   r4   r6   r   r   é   rÚ   Útwins_pcpvt_baserÑ   )rß   rÜ   rÝ   r0   r0   r1   rß   Ô  s     
 
 

 
þrß   c              
   K   sV   t dddddgdddd	gd	d	ddgd
d	dd
gd	dddgd�}tdd| it |f|Ž—ŽS )Nr5   r‘   r’   rÙ   r”   r   r4   r6   r   r   é   rÚ   Útwins_pcpvt_largerÑ   )rá   rÜ   rÝ   r0   r0   r1   rá   Ü  s     
 
 

 
þrá   c                 K   s`   t dddddgddddgddddgddd	dgd
d
d
d
gddddgd�}tdd| it |f|Ž—ŽS )Nr5   r‘   r’   r“   r”   r4   r   rŠ   é
   é   r   ©rŒ   rž   r"   rŸ   r¥   r¢   r¡   Útwins_svt_smallrÑ   )rå   rÜ   rÝ   r0   r0   r1   rå   ä  s     
 
 

 
 
þrå   c                 K   s`   t dddddgdddd	gddddgd
d
dd
gddddgddd
dgd�}tdd| it |f|Ž—ŽS )Nr5   é`   éÀ   i€  r|   r   r•   é   é   r4   rÞ   rã   r   r   rä   Útwins_svt_baserÑ   )rê   rÜ   rÝ   r0   r0   r1   rê   ì  s     
 
 

 
 
þrê   c                 K   s`   t dddddgddddgddddgd	d	d
d	gddddgddd	dgd�}tdd| it |f|Ž—ŽS )Nr5   r’   r“   r”   i   r   rŠ   é    r4   rÞ   rã   r   rä   Útwins_svt_largerÑ   )rì   rÜ   rÝ   r0   r0   r1   rì   ô  s     
 
 

 
 
þrì   )F)r¼   )F)F)F)F)F)F)1rZ   rÃ   Ú	functoolsr   Útypingr   r[   Ztorch.nnr$   Ztorch.nn.functionalrl   r=   Z	timm.datar   r   Ztimm.layersr   r   r	   r
   r   Z_builderr   Z_features_fxr   Ú	_registryr   r   Zvision_transformerr   Ú__all__rt   r`   ÚModuler   rb   rm   r{   rˆ   r   rÔ   rØ   Zdefault_cfgsrÛ   rß   rá   rå   rê   rì   r0   r0   r0   r1   Ú<module>   sX   a;( 

ú
