U
    ¯Ê¹jÓF  ã                   @   s@  d Z ddlmZ ddlZddlmZ ddlmZmZ ddl	m
Z
mZmZmZmZ ddlmZ ddlmZ dd	lmZmZ d
ddddgZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd
„ d
ejƒZdAdd„ZdBdd„ZdCdd„Zeedddd�edd d!�edd"dd�edd#d!�edd$d!�edd%dd�edd&d!�edd'd!�edd(d!�edd)d*d�d+œ
ƒZ edDed,œd-d.„ƒZ!edEed,œd/d0„ƒZ"edFed,œd1d2„ƒZ#edGed,œd3d4„ƒZ$edHed,œd5d6„ƒZ%edIed,œd7d8„ƒZ&edJed,œd9d:„ƒZ'edKed,œd;d<„ƒZ(edLed,œd=d>„ƒZ)edMed,œd?d@„ƒZ*dS )Na3   Class-Attention in Image Transformers (CaiT)

Paper: 'Going deeper with Image Transformers' - https://arxiv.org/abs/2103.17239

Original code and weights from https://github.com/facebookresearch/deit, copyright below

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
é    )ÚpartialN©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Ú
PatchEmbedÚMlpÚDropPathÚtrunc_normal_Úuse_fused_attné   )Úbuild_model_with_cfg)Úcheckpoint_seq)Úregister_modelÚgenerate_default_cfgsÚCaitÚ	ClassAttnÚLayerScaleBlockClassAttnÚLayerScaleBlockÚTalkingHeadAttnc                       s8   e Zd ZU ejje ed< d	‡ fdd„	Zdd„ Z	‡  Z
S )
r   Ú
fused_attné   Fç        c                    sŠ   t ƒ  ¡  || _|| }|d | _tƒ | _tj|||d�| _tj|||d�| _	tj|||d�| _
t |¡| _t ||¡| _t |¡| _d S )Nç      à¿©Úbias)ÚsuperÚ__init__Ú	num_headsÚscaler
   r   ÚnnÚLinearÚqÚkÚvÚDropoutÚ	attn_dropÚprojÚ	proj_drop©ÚselfÚdimr   Úqkv_biasr%   r'   Zhead_dim©Ú	__class__© úO/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/cait.pyr      s    

zClassAttn.__init__c           
      C   s.  |j \}}}|  |d d …df ¡ d¡ |d| j|| j ¡ dddd¡}|  |¡ ||| j|| j ¡ dddd¡}|  |¡ ||| j|| j ¡ dddd¡}| jrÈt	j
jj|||| jr¾| jjndd�}n8|| j }|| dd¡ }	|	jdd	�}	|  |	¡}	|	| }| dd¡ |d|¡}|  |¡}|  |¡}|S )
Nr   r   é   é   r   )Z	dropout_péþÿÿÿéÿÿÿÿ©r*   )Úshaper!   Z	unsqueezeÚreshaper   Úpermuter"   r#   r   Útorchr   Z
functionalZscaled_dot_product_attentionZtrainingr%   Úpr   Ú	transposeÚsoftmaxr&   r'   )
r)   ÚxÚBÚNÚCr!   r"   r#   Úx_clsÚattnr.   r.   r/   Úforward,   s(    <**  þ



zClassAttn.forward)r   Fr   r   )Ú__name__Ú
__module__Ú__qualname__r8   ÚjitÚFinalÚboolÚ__annotations__r   rB   Ú__classcell__r.   r.   r,   r/   r      s   
c                
       s>   e Zd Zdddddejejeedf
‡ fdd„	Zdd„ Z	‡  Z
S )	r   ç      @Fr   ç-Cëâ6?c                    sš   t ƒ  ¡  |	|ƒ| _|
|||||d�| _|dkr8t|ƒnt ¡ | _|	|ƒ| _t	|| ƒ}|||||d�| _
t |t |¡ ¡| _t |t |¡ ¡| _d S ©N)r   r+   r%   r'   r   )Zin_featuresZhidden_featuresÚ	act_layerZdrop©r   r   Únorm1rA   r   r   ÚIdentityÚ	drop_pathÚnorm2ÚintÚmlpÚ	Parameterr8   ZonesÚgamma_1Úgamma_2©r)   r*   r   Ú	mlp_ratior+   r'   r%   rR   rN   Ú
norm_layerÚ
attn_blockÚ	mlp_blockÚinit_valuesZmlp_hidden_dimr,   r.   r/   r   H   s(    

û
üz!LayerScaleBlockClassAttn.__init__c              	   C   sV   t j||fdd�}||  | j|  |  |¡¡ ¡ }||  | j|  |  |¡¡ ¡ }|S )Nr   r4   )	r8   ÚcatrR   rW   rA   rP   rX   rU   rS   )r)   r<   r@   Úur.   r.   r/   rB   l   s      z LayerScaleBlockClassAttn.forward)rC   rD   rE   r   ÚGELUÚ	LayerNormr   r   r   rB   rJ   r.   r.   r,   r/   r   E   s   ó$c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )	r   r   Fr   c                    s~   t ƒ  ¡  || _|| }|d | _tj||d |d�| _t |¡| _t ||¡| _	t ||¡| _
t ||¡| _t |¡| _d S )Nr   r1   r   )r   r   r   r   r   r    Úqkvr$   r%   r&   Úproj_lÚproj_wr'   r(   r,   r.   r/   r   v   s    

zTalkingHeadAttn.__init__c           
      C   sø   |j \}}}|  |¡ ||d| j|| j ¡ ddddd¡}|d | j |d |d   }}}|| dd¡ }	|  |	 dddd¡¡ dddd¡}	|	jdd�}	|  	|	 dddd¡¡ dddd¡}	|  
|	¡}	|	|  dd¡ |||¡}|  |¡}|  |¡}|S )	Nr1   r0   r   r   é   r2   r3   r4   )r5   rc   r6   r   r7   r   r:   rd   r;   re   r%   r&   r'   )
r)   r<   r=   r>   r?   rc   r!   r"   r#   rA   r.   r.   r/   rB   ‰   s    ."""


zTalkingHeadAttn.forward)r   Fr   r   )rC   rD   rE   r   rB   rJ   r.   r.   r,   r/   r   s   s   c                
       s>   e Zd Zdddddejejeedf
‡ fdd„	Zdd„ Z	‡  Z
S )	r   rK   Fr   rL   c                    sš   t ƒ  ¡  |	|ƒ| _|
|||||d�| _|dkr8t|ƒnt ¡ | _|	|ƒ| _t	|| ƒ}|||||d�| _
t |t |¡ ¡| _t |t |¡ ¡| _d S rM   rO   rY   r,   r.   r/   r       s(    

û
üzLayerScaleBlock.__init__c              	   C   sD   ||   | j|  |  |¡¡ ¡ }||   | j|  |  |¡¡ ¡ }|S ©N)rR   rW   rA   rP   rX   rU   rS   ©r)   r<   r.   r.   r/   rB   Ä   s      zLayerScaleBlock.forward)rC   rD   rE   r   ra   rb   r   r   r   rB   rJ   r.   r.   r,   r/   r   �   s   ó$c                       sÖ   e Zd Zdddddddddd	d
d
d
d
d
eeeeejdd�ej	e
edeeddf‡ fdd„	Zdd„ Zejjdd„ ƒZejjd&dd„ƒZejjd'dd„ƒZejjdd„ ƒZd(dd„Zdd „ Zd)ed!œd"d#„Zd$d%„ Z‡  ZS )*r   éà   é   r1   éè  Útokené   é   rK   Tr   ç�íµ ÷Æ°>)ÚepsrL   r0   c                    sx  t ƒ  ¡  |dkst‚|| _|| _ˆ | _| _d| _||||ˆd�| _| jj	}t
 t ddˆ¡¡| _t
 t d|ˆ¡¡| _t
j|d�| _‡fdd„t|ƒD ƒ‰t
j‡ ‡‡‡‡‡‡	‡
‡‡‡‡‡fdd„t|ƒD ƒŽ | _t
 ‡ ‡‡‡‡	‡‡‡‡‡f
d	d„t|ƒD ƒ¡| _ˆˆƒ| _tˆd
dd�g| _t
 |¡| _|d
k�rBt
 ˆ|¡nt
 ¡ | _t| jdd� t| jdd� |  | j ¡ d S )N©Ú rl   ÚavgF)Úimg_sizeÚ
patch_sizeÚin_chansÚ	embed_dimr   )r9   c                    s   g | ]}ˆ ‘qS r.   r.   ©Ú.0Úi)Údrop_path_rater.   r/   Ú
<listcomp>   s     z!Cait.__init__.<locals>.<listcomp>c                    s0   g | ](}ˆˆˆ
ˆˆˆˆˆ| ˆ	ˆ ˆˆˆd �‘qS ))r*   r   rZ   r+   r'   r%   rR   r[   rN   r\   r]   r^   r.   rx   )rN   r\   Úattn_drop_rateÚblock_layersÚdprrw   r^   r]   rZ   r[   r   Úproj_drop_rater+   r.   r/   r|     s   óôc                    s&   g | ]}ˆˆˆˆˆ	ˆˆ ˆˆˆd �	‘qS ))	r*   r   rZ   r+   r[   rN   r\   r]   r^   r.   )ry   Ú_)
rN   Úattn_block_token_onlyÚblock_layers_tokenrw   r^   Úmlp_block_token_onlyÚmlp_ratio_token_onlyr[   r   r+   r.   r/   r|     s   
ö÷r   Úhead)Znum_chsZ	reductionÚmoduleç{®Gáz”?©Ústd)!r   r   ÚAssertionErrorÚnum_classesÚglobal_poolÚnum_featuresrw   Úgrad_checkpointingÚpatch_embedÚnum_patchesr   rV   r8   ZzerosÚ	cls_tokenÚ	pos_embedr$   Úpos_dropÚrangeZ
SequentialÚblocksZ
ModuleListÚblocks_token_onlyÚnormÚdictZfeature_infoÚ	head_dropr    rQ   r†   r	   ÚapplyÚ_init_weights)r)   rt   ru   rv   rŒ   r�   rw   Údepthr   rZ   r+   Z	drop_rateZpos_drop_rater€   r}   r{   r~   rƒ   Zpatch_layerr[   rN   r\   r]   r^   r‚   r„   Zdepth_token_onlyr…   r‘   r,   )rN   r\   r‚   r}   r~   rƒ   r   r{   rw   r^   r]   r„   rZ   r…   r[   r   r€   r+   r/   r   Í   s<    
ü&ó
 
ö

 zCait.__init__c                 C   sr   t |tjƒrBt|jdd� t |tjƒrn|jd k	rntj |jd¡ n,t |tjƒrntj |jd¡ tj |jd¡ d S )Nrˆ   r‰   r   ç      ð?)	Ú
isinstancer   r    r	   Úweightr   ÚinitZ	constant_rb   )r)   Úmr.   r.   r/   rœ   &  s    zCait._init_weightsc                 C   s   ddhS )Nr“   r’   r.   ©r)   r.   r.   r/   Úno_weight_decay/  s    zCait.no_weight_decayc                 C   s
   || _ d S rg   )r�   )r)   Úenabler.   r.   r/   Úset_grad_checkpointing3  s    zCait.set_grad_checkpointingFc                    s   ‡ fdd„}|S )Nc                    s’   t ‡ fdd„dD ƒƒrdS ˆ  d¡r:tˆ  d¡d ƒd S ˆ  d¡rrtˆjƒtˆjƒ d }tˆ  d¡d ƒ| S ˆ  d	¡r†tˆjƒS td
ƒS d S )Nc                    s   g | ]}ˆ   |¡‘qS r.   )Ú
startswith)ry   Ún©Únamer.   r/   r|   :  s     z8Cait.group_matcher.<locals>._matcher.<locals>.<listcomp>)r’   r“   r�   r   zblocks.Ú.r   zblocks_token_only.znorm.Úinf)Úanyr§   rT   ÚsplitÚlenr–   r—   Úfloat)rª   Z	to_offsetr£   r©   r/   Ú_matcher9  s    



z$Cait.group_matcher.<locals>._matcherr.   )r)   Zcoarser±   r.   r£   r/   Úgroup_matcher7  s    zCait.group_matcherc                 C   s   | j S rg   )r†   r£   r.   r.   r/   Úget_classifierH  s    zCait.get_classifierNc                 C   sD   || _ |d k	r |dkst‚|| _|dkr6t | j|¡nt ¡ | _d S )Nrq   r   )rŒ   r‹   r�   r   r    rŽ   rQ   r†   )r)   rŒ   r�   r.   r.   r/   Úreset_classifierL  s
    zCait.reset_classifierc                 C   sš   |   |¡}|| j }|  |¡}| jr<tj ¡ s<t| j|ƒ}n
|  |¡}| j	 
|jd dd¡}t| jƒD ]\}}|||ƒ}qftj||fdd�}|  |¡}|S )Nr   r3   r   r4   )r�   r“   r”   r�   r8   rF   Zis_scriptingr   r–   r’   Úexpandr5   Ú	enumerater—   r_   r˜   )r)   r<   Z
cls_tokensrz   Zblkr.   r.   r/   Úforward_featuresS  s    




zCait.forward_features)Ú
pre_logitsc                 C   sX   | j r<| j dkr,|d d …dd …f jdd�n|d d …df }|  |¡}|rN|S |  |¡S )Nrs   r   r4   r   )r�   Úmeanrš   r†   )r)   r<   r¸   r.   r.   r/   Úforward_headb  s    6
zCait.forward_headc                 C   s   |   |¡}|  |¡}|S rg   )r·   rº   rh   r.   r.   r/   rB   h  s    

zCait.forward)T)F)N)F)rC   rD   rE   r   r   r   r   r   rb   ra   r   r   r   r   rœ   r8   rF   Úignorer¤   r¦   r²   r³   r´   r·   rH   rº   rB   rJ   r.   r.   r,   r/   r   Ê   sR   äY	


c                 C   s:   d| kr| d } i }|   ¡ D ]\}}||| dd¡< q|S )NÚmodelzmodule.rr   )ÚitemsÚreplace)Z
state_dictr¼   Zcheckpoint_no_moduler"   r#   r.   r.   r/   Úcheckpoint_filter_fnn  s    r¿   Fc                 K   s0   |  dd ¡rtdƒ‚tt| |fdti|—Ž}|S )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.Zpretrained_filter_fn)ÚgetÚRuntimeErrorr   r   r¿   )ÚvariantÚ
pretrainedÚkwargsr¼   r.   r.   r/   Ú_create_caitw  s    ýüûrÅ   rr   c                 K   s    | ddd dddt tdddœ|–S )	Nrk   )r1   é€  rÆ   rž   ZbicubicTzpatch_embed.projr†   )ÚurlrŒ   Ú
input_sizeZ	pool_sizeZcrop_pctÚinterpolationZfixed_input_sizer¹   rŠ   Z
first_convÚ
classifierr   )rÇ   rÄ   r.   r.   r/   Ú_cfg…  s          ûúrË   ztimm/z1https://dl.fbaipublicfiles.com/deit/XXS24_224.pth)r1   ri   ri   )Ú	hf_hub_idrÇ   rÈ   z1https://dl.fbaipublicfiles.com/deit/XXS24_384.pth)rÌ   rÇ   z1https://dl.fbaipublicfiles.com/deit/XXS36_224.pthz1https://dl.fbaipublicfiles.com/deit/XXS36_384.pthz0https://dl.fbaipublicfiles.com/deit/XS24_384.pthz/https://dl.fbaipublicfiles.com/deit/S24_224.pthz/https://dl.fbaipublicfiles.com/deit/S24_384.pthz/https://dl.fbaipublicfiles.com/deit/S36_384.pthz/https://dl.fbaipublicfiles.com/deit/M36_384.pthz/https://dl.fbaipublicfiles.com/deit/M48_448.pth)r1   éÀ  rÍ   )
zcait_xxs24_224.fb_dist_in1kzcait_xxs24_384.fb_dist_in1kzcait_xxs36_224.fb_dist_in1kzcait_xxs36_384.fb_dist_in1kzcait_xs24_384.fb_dist_in1kzcait_s24_224.fb_dist_in1kzcait_s24_384.fb_dist_in1kzcait_s36_384.fb_dist_in1kzcait_m36_384.fb_dist_in1kzcait_m48_448.fb_dist_in1k)Úreturnc                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
Nrj   éÀ   é   rf   çñhãˆµøä>©ru   rw   r�   r   r^   Úcait_xxs24_224rÃ   )rÓ   ©r™   rÅ   ©rÃ   rÄ   Z
model_argsr¼   r.   r.   r/   rÓ   À  s    rÓ   c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
Nrj   rÏ   rÐ   rf   rÑ   rÒ   Úcait_xxs24_384rÃ   )rÖ   rÔ   rÕ   r.   r.   r/   rÖ   Ç  s    rÖ   c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
Nrj   rÏ   é$   rf   rÑ   rÒ   Úcait_xxs36_224rÃ   )rØ   rÔ   rÕ   r.   r.   r/   rØ   Î  s    rØ   c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
Nrj   rÏ   r×   rf   rÑ   rÒ   Úcait_xxs36_384rÃ   )rÙ   rÔ   rÕ   r.   r.   r/   rÙ   Õ  s    rÙ   c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
Nrj   i   rÐ   é   rÑ   rÒ   Úcait_xs24_384rÃ   )rÛ   rÔ   rÕ   r.   r.   r/   rÛ   Ü  s    rÛ   c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
Nrj   rÆ   rÐ   r   rÑ   rÒ   Úcait_s24_224rÃ   )rÜ   rÔ   rÕ   r.   r.   r/   rÜ   ã  s    rÜ   c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
Nrj   rÆ   rÐ   r   rÑ   rÒ   Úcait_s24_384rÃ   )rÝ   rÔ   rÕ   r.   r.   r/   rÝ   ê  s    rÝ   c                 K   s0   t dddddd�}td	d| it |f|Ž—Ž}|S )
Nrj   rÆ   r×   r   ro   rÒ   Úcait_s36_384rÃ   )rÞ   rÔ   rÕ   r.   r.   r/   rÞ   ñ  s    rÞ   c                 K   s0   t dddddd�}tdd| it |f|Ž—Ž}|S )	Nrj   rm   r×   ro   rÒ   Úcait_m36_384rÃ   )rß   rÔ   rÕ   r.   r.   r/   rß   ø  s    rß   c                 K   s0   t dddddd�}tdd| it |f|Ž—Ž}|S )	Nrj   rm   é0   ro   rÒ   Úcait_m48_448rÃ   )rá   rÔ   rÕ   r.   r.   r/   rá   ÿ  s    rá   )N)F)rr   )F)F)F)F)F)F)F)F)F)F)+Ú__doc__Ú	functoolsr   r8   Ztorch.nnr   Z	timm.datar   r   Ztimm.layersr   r   r   r	   r
   Z_builderr   Z_manipulater   Ú	_registryr   r   Ú__all__ÚModuler   r   r   r   r   r¿   rÅ   rË   Zdefault_cfgsrÓ   rÖ   rØ   rÙ   rÛ   rÜ   rÝ   rÞ   rß   rá   r.   r.   r.   r/   Ú<module>   s¨   
,.*- %
	

ýþýþþýþþþýØ0