U
    ¯Ê¹j”  ã                /   @   s�  d Z ddlZddlmZ ddlZddlmZ ddlmZ ddl	m
Z
mZ ddlmZmZmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZ dgZeG dd„ dejƒƒZdªdd„ZG dd„ dejƒZ G dd„ dejƒZ!G dd„ dejƒZ"G dd„ dejƒZ#G dd„ dejƒZ$G dd„ dejƒZ%dd„ Z&d«d d!„Z'd¬d#d$„Z(ee(d%d&d'�e(d%d(d'�e(d%d)d*d+�e(d%d,d'�e(d%d-d'�e(d%d.d*d+�e(d%d/d'�e(d%d0d'�e(d%d1d*d+�e(d%d2d'�e(d%d3d'�e(d%d4d*d+�e(d%d5d'�e(d%d6d'�e(d%d7d*d+�e(d%d8d'�e(d%d9d'�e(d%d:d*d+�e(d%d;d'�e(d%d<d'�e(d%d=d*d+�e(d%d>d'�e(d%d?d'�e(d%d@d*d+�e(d%dAd'�e(d%dBd'�e(d%dCd*d+�e(d%dDd'�e(d%dEd'�e(d%dFd*d+�e(d%dGd'�e(d%dHd'�e(d%dId*d+�e(d%dJd'�e(d%dKd'�e(d%dLd*d+�e(d%dMd'�e(d%dNd'�e(d%dOd*d+�e(d%dPd'�e(d%dQd'�e(d%dRd*d+�dSœ*ƒZ)ed­e%dTœdUdV„ƒZ*ed®e%dTœdWdX„ƒZ+ed¯e%dTœdYdZ„ƒZ,ed°e%dTœd[d\„ƒZ-ed±e%dTœd]d^„ƒZ.ed²e%dTœd_d`„ƒZ/ed³e%dTœdadb„ƒZ0ed´e%dTœdcdd„ƒZ1edµe%dTœdedf„ƒZ2ed¶e%dTœdgdh„ƒZ3ed·e%dTœdidj„ƒZ4ed¸e%dTœdkdl„ƒZ5ed¹e%dTœdmdn„ƒZ6edºe%dTœdodp„ƒZ7ed»e%dTœdqdr„ƒZ8ed¼e%dTœdsdt„ƒZ9ed½e%dTœdudv„ƒZ:ed¾e%dTœdwdx„ƒZ;ed¿e%dTœdydz„ƒZ<edÀe%dTœd{d|„ƒZ=edÁe%dTœd}d~„ƒZ>edÂe%dTœdd€„ƒZ?edÃe%dTœd�d‚„ƒZ@edÄe%dTœdƒd„„ƒZAedÅe%dTœd…d†„ƒZBedÆe%dTœd‡dˆ„ƒZCedÇe%dTœd‰dŠ„ƒZDedÈe%dTœd‹dŒ„ƒZEeeFd�dŽd�d�d‘d’d“d”d•d–d—d˜d™dšd›dœd�dždŸd d¡d¢d£d¤d¥d¦d§d¨d©œƒ dS )Éa]   Cross-Covariance Image Transformer (XCiT) in PyTorch

Paper:
    - https://arxiv.org/abs/2106.09681

Same as the official implementation, with some minor adaptations, original copyright below
    - https://github.com/facebookresearch/xcit/blob/master/xcit.py

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
é    N)Úpartial)Ú
checkpoint©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚtrunc_normal_Ú	to_2tupleé   )Úbuild_model_with_cfg)Úregister_notrace_module)Úregister_modelÚgenerate_default_cfgsÚregister_model_deprecations)Ú	ClassAttn)ÚMlpÚXcitc                       s4   e Zd ZdZd
‡ fdd„	Zeeedœdd	„Z‡  ZS )ÚPositionalEncodingFourierzä
    Positional encoding relying on a fourier kernel matching the one used in the "Attention is all you Need" paper.
    Based on the official XCiT code
        - https://github.com/facebookresearch/xcit/blob/master/xcit.py
    é    é   é'  c                    sH   t ƒ  ¡  tj|d |dd�| _dtj | _|| _|| _	|| _
d| _d S )Né   r
   )Úkernel_sizeç�íµ ÷Æ°>)ÚsuperÚ__init__ÚnnÚConv2dÚtoken_projectionÚmathÚpiÚscaleÚtemperatureÚ
hidden_dimÚdimÚeps)Úselfr#   r$   r"   ©Ú	__class__© úO/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/xcit.pyr   (   s    
z"PositionalEncodingFourier.__init__)ÚBÚHÚWc              	   C   s(  | j jj}| j jj}tjd|d |d� tj¡ d¡ 	dd|¡}tjd|d |d� tj¡ 	d|d¡}||d d …dd …d d …f | j
  | j }||d d …d d …dd …f | j
  | j }tj| j|d� tj¡}| jdtj|ddd� | j  }|d d …d d …d d …d f | }	|d d …d d …d d …d f | }
tj|	d d …d d …d d …dd d…f  ¡ |	d d …d d …d d …dd d…f  ¡ gdd	� d
¡}	tj|
d d …d d …d d …dd d…f  ¡ |
d d …d d …d d …dd d…f  ¡ gdd	� d
¡}
tj|
|	fd
d	� dd
dd¡}|   | |¡¡}| 	|ddd¡S )Nr
   )Údeviceéÿÿÿÿr   Úfloor)Zrounding_moder   é   ©r$   é   )r   Úweightr.   ÚdtypeÚtorchZarangeÚtoZfloat32Z	unsqueezeÚrepeatr%   r!   r#   r"   ÚdivÚstackÚsinÚcosÚflattenÚcatÚpermute)r&   r+   r,   r-   r.   r5   Zy_embedZx_embedZdim_tZpos_xZpos_yÚposr)   r)   r*   Úforward1   s    

,&**   \\z!PositionalEncodingFourier.forward)r   r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚintrA   Ú__classcell__r)   r)   r'   r*   r       s   	r   c              
   C   s&   t j tj| |d|ddd�t |¡¡S )z3x3 convolution + batch normr3   r
   F)r   ÚstrideÚpaddingÚbias)r6   r   Ú
Sequentialr   ÚBatchNorm2d)Z	in_planesZ
out_planesrH   r)   r)   r*   Úconv3x3C   s    þrM   c                       s6   e Zd ZdZddddejf‡ fdd„	Zdd	„ Z‡  ZS )
ÚConvPatchEmbedz<Image to Patch Embedding using multiple convolutional layerséà   é   r3   r   c                    sò   t ƒ  ¡  t|ƒ}|d | |d |  }|| _|| _|| _|dkržtj t	||d dƒ|ƒ t	|d |d dƒ|ƒ t	|d |d dƒ|ƒ t	|d |dƒ¡| _
nP|dkrêtj t	||d dƒ|ƒ t	|d |d dƒ|ƒ t	|d |dƒ¡| _
nd‚d S )Nr
   r   rP   é   r   r1   z=For convolutional projection, patch size has to be in [8, 16])r   r   r	   Úimg_sizeÚ
patch_sizeÚnum_patchesr6   r   rK   rM   Úproj)r&   rR   rS   Úin_chansÚ	embed_dimÚ	act_layerrT   r'   r)   r*   r   N   s2    
ù	ûzConvPatchEmbed.__init__c                 C   s>   |   |¡}|jd |jd  }}| d¡ dd¡}|||ffS )Nr   r3   r
   )rU   Úshaper=   Ú	transpose)r&   ÚxÚHpÚWpr)   r)   r*   rA   k   s    
zConvPatchEmbed.forward)	rB   rC   rD   rE   r   ÚGELUr   rA   rG   r)   r)   r'   r*   rN   K   s   rN   c                       s:   e Zd ZdZdejdf‡ fdd„	Zeedœdd„Z‡  Z	S )	ÚLPIa  
    Local Patch Interaction module that allows explicit communication between tokens in 3x3 windows to augment the
    implicit communication performed by the block diagonal scatter attention. Implemented using 2 layers of separable
    3x3 convolutions with GeLU and BatchNorm2d
    Nr3   c                    sb   t ƒ  ¡  |p|}|d }tjj|||||d�| _|ƒ | _t |¡| _tjj|||||d�| _	d S )Nr   )r   rI   Úgroups)
r   r   r6   r   r   Úconv1ÚactrL   ÚbnÚconv2)r&   Úin_featuresZout_featuresrX   r   rI   r'   r)   r*   r   y   s&    
    ÿ    ÿzLPI.__init__©r,   r-   c                 C   sj   |j \}}}| ddd¡ ||||¡}|  |¡}|  |¡}|  |¡}|  |¡}| |||¡ ddd¡}|S )Nr   r   r
   )rY   r?   Úreshapera   rb   rc   rd   )r&   r[   r,   r-   r+   ÚNÚCr)   r)   r*   rA   †   s    



zLPI.forward)
rB   rC   rD   rE   r   r^   r   rF   rA   rG   r)   r)   r'   r*   r_   r   s   r_   c                	       s@   e Zd ZdZdddddejejddf	‡ fdd„	Zdd	„ Z‡  Z	S )
ÚClassAttentionBlockzAClass Attention Layer as in CaiT https://arxiv.org/abs/2103.17239ç      @Fç        ç      ð?c                    s²   t ƒ  ¡  |	|ƒ| _t|||||d�| _|dkr8t|ƒnt ¡ | _|	|ƒ| _	t
|t|| ƒ||d�| _|
d k	rœt |
t |¡ ¡| _t |
t |¡ ¡| _nd\| _| _|| _d S )N©Ú	num_headsÚqkv_biasÚ	attn_dropÚ	proj_droprl   ©re   Zhidden_featuresrX   Zdrop)rm   rm   )r   r   Únorm1r   Úattnr   r   ÚIdentityÚ	drop_pathÚnorm2r   rF   ÚmlpÚ	Parameterr6   ÚonesÚgamma1Úgamma2Útokens_norm)r&   r$   ro   Ú	mlp_ratiorp   rr   rq   rw   rX   Ú
norm_layerÚetar~   r'   r)   r*   r   ”   s"    

    ÿ
zClassAttentionBlock.__init__c                 C   sì   |   |¡}tj|  |¡|d d …dd …f gdd�}||  | j| ¡ }| jrX|  |¡}n8tj|  |d d …dd…f ¡|d d …dd …f gdd�}|}|d d …dd…f }| j|  	|¡ }tj||d d …dd …f gdd�}||  |¡ }|S )Nr
   r2   r   )
rt   r6   r>   ru   rw   r|   r~   rx   r}   ry   )r&   r[   Zx_norm1Zx_attnZx_resÚ	cls_tokenr)   r)   r*   rA   µ   s    
(8"zClassAttentionBlock.forward)
rB   rC   rD   rE   r   r^   Ú	LayerNormr   rA   rG   r)   r)   r'   r*   rj   ‘   s   ô!rj   c                       s:   e Zd ZdZd‡ fdd„	Zdd„ Zejjd	d
„ ƒZ	‡  Z
S )ÚXCAzÝ Cross-Covariance Attention (XCA)
    Operation where the channels are updated using a weighted sum. The weights are obtained from the (softmax
    normalized) Cross-covariance matrix (Q^T \cdot K \in d_h \times d_h)
    rQ   Frl   c                    sf   t ƒ  ¡  || _t t |dd¡¡| _tj||d |d�| _	t 
|¡| _t ||¡| _t 
|¡| _d S )Nr
   r3   )rJ   )r   r   ro   r   rz   r6   r{   r"   ÚLinearÚqkvÚDropoutrq   rU   rr   )r&   r$   ro   rp   rq   rr   r'   r)   r*   r   Ë   s    
zXCA.__init__c           
      C   sÐ   |j \}}}|  |¡ ||d| j|| j ¡ ddddd¡}| d¡\}}}tjjj	|dd�}tjjj	|dd�}|| 
dd¡ | j }	|	jdd�}	|  |	¡}	|	|  dddd¡ |||¡}|  |¡}|  |¡}|S )	Nr3   r   r   r1   r
   r/   r2   éþÿÿÿ)rY   r†   rg   ro   r?   Zunbindr6   r   Z
functionalÚ	normalizerZ   r"   Zsoftmaxrq   rU   rr   )
r&   r[   r+   rh   ri   r†   ÚqÚkÚvru   r)   r)   r*   rA   Ô   s    .


zXCA.forwardc                 C   s   dhS )Nr"   r)   ©r&   r)   r)   r*   Úno_weight_decayç   s    zXCA.no_weight_decay)rQ   Frl   rl   )rB   rC   rD   rE   r   rA   r6   ÚjitÚignorerŽ   rG   r)   r)   r'   r*   r„   Å   s
   	r„   c                       sB   e Zd Zdddddejejdf‡ fdd„	Zeedœdd	„Z‡  Z	S )
ÚXCABlockrk   Frl   rm   c                    sÄ   t ƒ  ¡  |	|ƒ| _t|||||d�| _|dkr8t|ƒnt ¡ | _|	|ƒ| _	t
||d�| _|	|ƒ| _t|t|| ƒ||d�| _t |
t |¡ ¡| _t |
t |¡ ¡| _t |
t |¡ ¡| _d S )Nrn   rl   )re   rX   rs   )r   r   rt   r„   ru   r   r   rv   rw   Únorm3r_   Úlocal_mprx   r   rF   ry   rz   r6   r{   r|   Úgamma3r}   )r&   r$   ro   r   rp   rr   rq   rw   rX   r€   r�   r'   r)   r*   r   í   s    



zXCABlock.__init__rf   c              	   C   sh   ||   | j|  |  |¡¡ ¡ }||   | j|  |  |¡||¡ ¡ }||   | j|  |  	|¡¡ ¡ }|S ©N)
rw   r|   ru   rt   r”   r“   r’   r}   ry   rx   )r&   r[   r,   r-   r)   r)   r*   rA   	  s     $ zXCABlock.forward)
rB   rC   rD   r   r^   rƒ   r   rF   rA   rG   r)   r)   r'   r*   r‘   ì   s   õr‘   c                       s˜   e Zd ZdZd&‡ fdd„	Zdd„ Zejjdd„ ƒZ	ejjd'dd„ƒZ
ejjd(dd„ƒZejjdd„ ƒZd)dd„Zdd „ Zd*ed!œd"d#„Zd$d%„ Z‡  ZS )+r   z 
    Based on timm and DeiT code bases
    https://github.com/rwightman/pytorch-image-models/tree/master/timm
    https://github.com/facebookresearch/deit/
    rO   rP   r3   éè  Útokenr   é   rk   Trl   Nr   rm   Fc                    s†  t ƒ  ¡  |dkst‚t|ƒ}|d | dkr>|d | dksFtdƒ‚ˆpVttjdd�‰ˆ p`tj‰ || _ˆ | _	| _
|| _d| _t|||ˆˆ d�| _t t ddˆ¡¡| _|r¼tˆd	�| _nd
| _tj|d�| _t ‡ ‡‡‡‡‡‡‡‡	‡
f
dd„t|ƒD ƒ¡| _t ‡ ‡‡‡‡‡‡‡‡
‡f
dd„t|ƒD ƒ¡| _ˆˆƒ| _t ˆ¡| _|dk�r^t | j	|¡nt ¡ | _t | jdd� |  !| j"¡ d
S )a¦  
        Args:
            img_size (int, tuple): input image size
            patch_size (int): patch size
            in_chans (int): number of input channels
            num_classes (int): number of classes for classification head
            embed_dim (int): embedding dimension
            depth (int): depth of transformer
            num_heads (int): number of attention heads
            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
            qkv_bias (bool): enable bias for qkv if True
            drop_rate (float): dropout rate after positional embedding, and in XCA/CA projection + MLP
            pos_drop_rate: position embedding dropout rate
            proj_drop_rate (float): projection dropout rate
            attn_drop_rate (float): attention dropout rate
            drop_path_rate (float): stochastic depth rate (constant across all layers)
            norm_layer: (nn.Module): normalization layer
            cls_attn_layers: (int) Depth of Class attention layers
            use_pos_embed: (bool) whether to use positional encoding
            eta: (float) layerscale initialization value
            tokens_norm: (bool) Whether to normalize all tokens or just the cls_token in the CA

        Notes:
            - Although `layer_norm` is user specifiable, there are hard-coded `BatchNorm2d`s in the local patch
              interaction (class LPI) and the patch embedding (class ConvPatchEmbed)
        ©Ú Úavgr—   r   z2`patch_size` should divide image dimensions evenlyr   )r%   F)rR   rS   rV   rW   rX   r
   r2   N)Úpc                    s(   g | ] }t ˆˆˆˆ	ˆˆˆˆ ˆˆd �
‘qS ))
r$   ro   r   rp   rr   rq   rw   rX   r€   r�   )r‘   ©Ú.0Ú_)
rX   Úattn_drop_rateÚdrop_path_raterW   r�   r   r€   ro   Úproj_drop_raterp   r)   r*   Ú
<listcomp>g  s   ôöz!Xcit.__init__.<locals>.<listcomp>c                    s(   g | ] }t ˆˆˆˆˆˆˆ ˆˆˆ	d �
‘qS ))
r$   ro   r   rp   rr   rq   rX   r€   r�   r~   )rj   r�   )
rX   r    Ú	drop_raterW   r�   r   r€   ro   rp   r~   r)   r*   r£   v  s   ôöç{®Gáz”?©Ústd)#r   r   ÚAssertionErrorr	   r   r   rƒ   r^   Únum_classesÚnum_featuresrW   Úglobal_poolÚgrad_checkpointingrN   Úpatch_embedrz   r6   Zzerosr‚   r   Ú	pos_embedr‡   Úpos_dropZ
ModuleListÚrangeÚblocksÚcls_attn_blocksÚnormÚ	head_dropr…   rv   Úheadr   ÚapplyÚ_init_weights)r&   rR   rS   rV   r©   r«   rW   Údepthro   r   rp   r¤   Zpos_drop_rater¢   r    r¡   rX   r€   Zcls_attn_layersÚuse_pos_embedr�   r~   r'   )rX   r    r¡   r¤   rW   r�   r   r€   ro   r¢   rp   r~   r*   r     sF    2
"ÿ
û ó
 ó

"zXcit.__init__c                 C   sD   t |tjƒr@t|jdd� t |tjƒr@|jd k	r@tj |jd¡ d S )Nr¥   r¦   r   )Ú
isinstancer   r…   r   r4   rJ   ÚinitZ	constant_)r&   Úmr)   r)   r*   r·   Ž  s    zXcit._init_weightsc                 C   s   ddhS )Nr®   r‚   r)   r�   r)   r)   r*   rŽ   ”  s    zXcit.no_weight_decayc                 C   s   t ddddgd�S )Nz ^cls_token|pos_embed|patch_embedz^blocks\.(\d+))z^cls_attn_blocks\.(\d+)N)z^norm)iŸ† )Ústemr±   r²   )Údict)r&   Zcoarser)   r)   r*   Úgroup_matcher˜  s
    ýzXcit.group_matcherc                 C   s
   || _ d S r•   )r¬   )r&   Úenabler)   r)   r*   Úset_grad_checkpointing   s    zXcit.set_grad_checkpointingc                 C   s   | j S r•   )rµ   r�   r)   r)   r*   Úget_classifier¤  s    zXcit.get_classifierrš   c                 C   sD   || _ |d k	r |dkst‚|| _|dkr6t | j|¡nt ¡ | _d S )Nr™   r   )r©   r¨   r«   r   r…   rª   rv   rµ   )r&   r©   r«   r)   r)   r*   Úreset_classifier¨  s
    zXcit.reset_classifierc                 C   sô   |j d }|  |¡\}\}}| jd k	rV|  |||¡ |d|j d ¡ ddd¡}|| }|  |¡}| jD ]0}| jrŠtj	 
¡ sŠt||||ƒ}qf||||ƒ}qftj| j |dd¡|fdd�}| jD ](}| jrÜtj	 
¡ sÜt||ƒ}q¼||ƒ}q¼|  |¡}|S )Nr   r/   r
   r   r2   )rY   r­   r®   rg   r?   r¯   r±   r¬   r6   r�   Zis_scriptingr   r>   r‚   Úexpandr²   r³   )r&   r[   r+   r\   r]   Zpos_encodingZblkr)   r)   r*   Úforward_features¯  s"    

(




zXcit.forward_features)Ú
pre_logitsc                 C   sX   | j r<| j dkr,|d d …dd …f jdd�n|d d …df }|  |¡}|rN|S |  |¡S )Nr›   r
   r2   r   )r«   Úmeanr´   rµ   )r&   r[   rÆ   r)   r)   r*   Úforward_headË  s    6
zXcit.forward_headc                 C   s   |   |¡}|  |¡}|S r•   )rÅ   rÈ   )r&   r[   r)   r)   r*   rA   Ñ  s    

zXcit.forward)rO   rP   r3   r–   r—   r   r˜   r˜   rk   Trl   rl   rl   rl   rl   NNr   Trm   F)F)T)rš   )F)rB   rC   rD   rE   r   r·   r6   r�   r�   rŽ   r¿   rÁ   rÂ   rÃ   rÅ   ÚboolrÈ   rA   rG   r)   r)   r'   r*   r     sH                        êu


c              	   C   s6  d| kr| d } t |dd ƒd k	}dd„ | D ƒ}|D ]&}|rR|  |¡| | dd¡< q2| |= q2d| k�r2d| ¡ k�r2t|jƒ}t|ƒD ]¬}|  d	|› d
�¡}| dd|jd ¡}t	dƒD ]"\}}	|| | d	|› d|	› d�< q¶|  d	|› d�d ¡}
|
d k	r„|
 dd¡}
t	dƒD ]$\}}	|
| | d	|› d|	› d�< �q
q„| S )NÚmodelr®   c                 S   s   g | ]}|  d ¡r|‘qS )r®   )Ú
startswith)rž   r‹   r)   r)   r*   r£   Ý  s     
 z(checkpoint_filter_fn.<locals>.<listcomp>zpos_embeder.z
pos_embed.z!cls_attn_blocks.0.attn.qkv.weightzcls_attn_blocks.0.attn.q.weightzcls_attn_blocks.z.attn.qkv.weightr3   r/   r†   z.attn.z.weightz.attn.qkv.biasz.bias)
ÚgetattrÚpopÚreplaceÚ
state_dictÚlenr²   r°   rg   rY   Ú	enumerate)rÏ   rÊ   r¹   Zpos_embed_keysr‹   Znum_ca_blocksÚiZ
qkv_weightÚjZ	subscriptrp   r)   r)   r*   Úcheckpoint_filter_fn×  s*    
 rÔ   Fc                 K   s0   |  dd ¡rtdƒ‚tt| |fdti|—Ž}|S )NZfeatures_onlyzMfeatures_only not implemented for Cross-Covariance Image Transformers models.Zpretrained_filter_fn)ÚgetÚRuntimeErrorr   r   rÔ   )ÚvariantÚ
pretrainedZdefault_cfgÚkwargsrÊ   r)   r)   r*   Ú_create_xcitô  s    ýüûrÚ   rš   c                 K   s    | ddd dddt tdddœ|–S )	Nr–   )r3   rO   rO   rm   ZbicubicTzpatch_embed.proj.0.0rµ   )Úurlr©   Ú
input_sizeZ	pool_sizeZcrop_pctÚinterpolationZfixed_input_sizerÇ   r§   Z
first_convÚ
classifierr   )rÛ   rÙ   r)   r)   r*   Ú_cfg  s          ûúrß   ztimm/z<https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_224.pth)Ú	hf_hub_idrÛ   zAhttps://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_384_dist.pth)r3   é€  rá   )rà   rÛ   rÜ   z<https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_384_dist.pthz<https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_384_dist.pthz=https://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_224.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_224_dist.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_384_dist.pthz=https://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_224.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_224_dist.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_384_dist.pthz>https://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_224.pthzChttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_224_dist.pthzChttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_384_dist.pthz=https://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_224.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_224_dist.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_384_dist.pthz;https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_224.pthz@https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_224_dist.pthz@https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_384_dist.pthz;https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_224.pthz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_224_dist.pthz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_384_dist.pthz;https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_224.pthz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_224_dist.pthz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_384_dist.pthz<https://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_384_dist.pthz<https://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_384_dist.pthz=https://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_224.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_224_dist.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_384_dist.pthz<https://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_384_dist.pth)*zxcit_nano_12_p16_224.fb_in1kú!xcit_nano_12_p16_224.fb_dist_in1kú!xcit_nano_12_p16_384.fb_dist_in1kzxcit_tiny_12_p16_224.fb_in1kú!xcit_tiny_12_p16_224.fb_dist_in1kú!xcit_tiny_12_p16_384.fb_dist_in1kzxcit_tiny_24_p16_224.fb_in1kú!xcit_tiny_24_p16_224.fb_dist_in1kú!xcit_tiny_24_p16_384.fb_dist_in1kzxcit_small_12_p16_224.fb_in1kú"xcit_small_12_p16_224.fb_dist_in1kú"xcit_small_12_p16_384.fb_dist_in1kzxcit_small_24_p16_224.fb_in1kú"xcit_small_24_p16_224.fb_dist_in1kú"xcit_small_24_p16_384.fb_dist_in1kzxcit_medium_24_p16_224.fb_in1kú#xcit_medium_24_p16_224.fb_dist_in1kú#xcit_medium_24_p16_384.fb_dist_in1kzxcit_large_24_p16_224.fb_in1kú"xcit_large_24_p16_224.fb_dist_in1kú"xcit_large_24_p16_384.fb_dist_in1kzxcit_nano_12_p8_224.fb_in1kú xcit_nano_12_p8_224.fb_dist_in1kú xcit_nano_12_p8_384.fb_dist_in1kzxcit_tiny_12_p8_224.fb_in1kú xcit_tiny_12_p8_224.fb_dist_in1kú xcit_tiny_12_p8_384.fb_dist_in1kzxcit_tiny_24_p8_224.fb_in1kú xcit_tiny_24_p8_224.fb_dist_in1kú xcit_tiny_24_p8_384.fb_dist_in1kzxcit_small_12_p8_224.fb_in1kú!xcit_small_12_p8_224.fb_dist_in1kú!xcit_small_12_p8_384.fb_dist_in1kzxcit_small_24_p8_224.fb_in1kú!xcit_small_24_p8_224.fb_dist_in1kú!xcit_small_24_p8_384.fb_dist_in1kzxcit_medium_24_p8_224.fb_in1kú"xcit_medium_24_p8_224.fb_dist_in1kú"xcit_medium_24_p8_384.fb_dist_in1kzxcit_large_24_p8_224.fb_in1kú!xcit_large_24_p8_224.fb_dist_in1kú!xcit_large_24_p8_384.fb_dist_in1k)Úreturnc                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrP   é€   r˜   r1   rm   F©rS   rW   r¸   ro   r�   r~   Úxcit_nano_12_p16_224rØ   )r  ©r¾   rÚ   ©rØ   rÙ   Z
model_argsrÊ   r)   r)   r*   r  ’  s         ÿr  c              	   K   s4   t dddddddd�}tdd
| it |f|Ž—Ž}|S )NrP   rÿ   r˜   r1   rm   Frá   )rS   rW   r¸   ro   r�   r~   rR   Úxcit_nano_12_p16_384rØ   )r  r  r  r)   r)   r*   r  š  s          ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrP   éÀ   r˜   r1   rm   Tr   Úxcit_tiny_12_p16_224rØ   )r  r  r  r)   r)   r*   r  ¢  s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrP   r  r˜   r1   rm   Tr   Úxcit_tiny_12_p16_384rØ   )r  r  r  r)   r)   r*   r  ª  s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrP   rá   r˜   rQ   rm   Tr   Úxcit_small_12_p16_224rØ   )r  r  r  r)   r)   r*   r  ²  s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrP   rá   r˜   rQ   rm   Tr   Úxcit_small_12_p16_384rØ   )r	  r  r  r)   r)   r*   r	  º  s         ÿr	  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrP   r  é   r1   çñhãˆµøä>Tr   Úxcit_tiny_24_p16_224rØ   )r  r  r  r)   r)   r*   r  Â  s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrP   r  r
  r1   r  Tr   Úxcit_tiny_24_p16_384rØ   )r  r  r  r)   r)   r*   r  Ê  s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrP   rá   r
  rQ   r  Tr   Úxcit_small_24_p16_224rØ   )r  r  r  r)   r)   r*   r  Ò  s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrP   rá   r
  rQ   r  Tr   Úxcit_small_24_p16_384rØ   )r  r  r  r)   r)   r*   r  Ú  s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrP   é   r
  rQ   r  Tr   Úxcit_medium_24_p16_224rØ   )r  r  r  r)   r)   r*   r  â  s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrP   r  r
  rQ   r  Tr   Úxcit_medium_24_p16_384rØ   )r  r  r  r)   r)   r*   r  ê  s         ÿr  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrP   r   r
  r  Tr   Úxcit_large_24_p16_224rØ   )r  r  r  r)   r)   r*   r  ò  s         ÿr  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrP   r   r
  r  Tr   Úxcit_large_24_p16_384rØ   )r  r  r  r)   r)   r*   r  ú  s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrQ   rÿ   r˜   r1   rm   Fr   Úxcit_nano_12_p8_224rØ   )r  r  r  r)   r)   r*   r    s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrQ   rÿ   r˜   r1   rm   Fr   Úxcit_nano_12_p8_384rØ   )r  r  r  r)   r)   r*   r    s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrQ   r  r˜   r1   rm   Tr   Úxcit_tiny_12_p8_224rØ   )r  r  r  r)   r)   r*   r    s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrQ   r  r˜   r1   rm   Tr   Úxcit_tiny_12_p8_384rØ   )r  r  r  r)   r)   r*   r    s         ÿr  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrQ   rá   r˜   rm   Tr   Úxcit_small_12_p8_224rØ   )r  r  r  r)   r)   r*   r  #  s         ÿr  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrQ   rá   r˜   rm   Tr   Úxcit_small_12_p8_384rØ   )r  r  r  r)   r)   r*   r  +  s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrQ   r  r
  r1   r  Tr   Úxcit_tiny_24_p8_224rØ   )r  r  r  r)   r)   r*   r  3  s         ÿr  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrQ   r  r
  r1   r  Tr   Úxcit_tiny_24_p8_384rØ   )r  r  r  r)   r)   r*   r  ;  s         ÿr  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrQ   rá   r
  r  Tr   Úxcit_small_24_p8_224rØ   )r  r  r  r)   r)   r*   r  C  s         ÿr  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrQ   rá   r
  r  Tr   Úxcit_small_24_p8_384rØ   )r  r  r  r)   r)   r*   r  K  s         ÿr  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrQ   r  r
  r  Tr   Úxcit_medium_24_p8_224rØ   )r  r  r  r)   r)   r*   r  S  s         ÿr  c                 K   s2   t ddddddd�}td	d| it |f|Ž—Ž}|S )
NrQ   r  r
  r  Tr   Úxcit_medium_24_p8_384rØ   )r   r  r  r)   r)   r*   r   [  s         ÿr   c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrQ   r   r
  rP   r  Tr   Úxcit_large_24_p8_224rØ   )r!  r  r  r)   r)   r*   r!  c  s         ÿr!  c                 K   s2   t ddddddd�}td
d	| it |f|Ž—Ž}|S )NrQ   r   r
  rP   r  Tr   Úxcit_large_24_p8_384rØ   )r"  r  r  r)   r)   r*   r"  k  s         ÿr"  râ   rã   rä   rå   ræ   rç   rè   ré   rê   rë   rì   rí   rî   rï   rð   rñ   rò   ró   rô   rõ   rö   r÷   rø   rù   rú   rû   rü   rý   )Zxcit_nano_12_p16_224_distZxcit_nano_12_p16_384_distZxcit_tiny_12_p16_224_distZxcit_tiny_12_p16_384_distZxcit_tiny_24_p16_224_distZxcit_tiny_24_p16_384_distZxcit_small_12_p16_224_distZxcit_small_12_p16_384_distZxcit_small_24_p16_224_distZxcit_small_24_p16_384_distZxcit_medium_24_p16_224_distZxcit_medium_24_p16_384_distZxcit_large_24_p16_224_distZxcit_large_24_p16_384_distZxcit_nano_12_p8_224_distZxcit_nano_12_p8_384_distZxcit_tiny_12_p8_224_distZxcit_tiny_12_p8_384_distZxcit_tiny_24_p8_224_distZxcit_tiny_24_p8_384_distZxcit_small_12_p8_224_distZxcit_small_12_p8_384_distZxcit_small_24_p8_224_distZxcit_small_24_p8_384_distZxcit_medium_24_p8_224_distZxcit_medium_24_p8_384_distZxcit_large_24_p8_224_distZxcit_large_24_p8_384_dist)r
   )FN)rš   )F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)GrE   r   Ú	functoolsr   r6   Ztorch.nnr   Ztorch.utils.checkpointr   Z	timm.datar   r   Ztimm.layersr   r   r	   Z_builderr   Z_features_fxr   Ú	_registryr   r   r   Zcaitr   Zvision_transformerr   Ú__all__ÚModuler   rM   rN   r_   rj   r„   r‘   r   rÔ   rÚ   rß   Zdefault_cfgsr  r  r  r  r  r	  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r!  r"  rB   r)   r)   r)   r*   Ú<module>   sR  "
'4'& F

þþ þþþ þþþ þþþ þþþ þþþ þþþ þþþ þþþ þþþ þþþ þþþ þþþ þþþ þ� á