U
    ¯Ê¹j‘S  ã                   @   sb  d Z ddlZddlmZ ddlmZmZmZmZm	Z	 ddl
Z
ddlmZ ddlm  mZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZ ddlm Z  dd	l!m"Z" dd
l#m$Z$ ddl%m&Z&m'Z' dgZ(G dd„ dej)ƒZ*G dd„ dej)ƒZ+G dd„ dej)ƒZ,G dd„ dej)ƒZ-G dd„ dej)ƒZ.ee/e/f dœdd„Z0e"ee/e/f ee/e/f dœdd„ƒZ1G dd„ dej)ƒZ2G dd „ d ej)ƒZ3G d!d"„ d"ej)ƒZ4G d#d„ dej)ƒZ5d<d%d&„Z6d=d(d)„Z7e'e7d*d+�e7d,d+�e7d-d+�e7d.d+�e7d/d+�d0œƒZ8e&d>e5d1œd2d3„ƒZ9e&d?e5d1œd4d5„ƒZ:e&d@e5d1œd6d7„ƒZ;e&dAe5d1œd8d9„ƒZ<e&dBe5d1œd:d;„ƒZ=dS )Ca)   Global Context ViT

From scratch implementation of GCViT in the style of timm swin_transformer_v2_cr.py

Global Context Vision Transformers -https://arxiv.org/abs/2206.09959

@article{hatamizadeh2022global,
  title={Global Context Vision Transformers},
  author={Hatamizadeh, Ali and Yin, Hongxu and Kautz, Jan and Molchanov, Pavlo},
  journal={arXiv preprint arXiv:2206.09959},
  year={2022}
}

Free of any code related to NVIDIA GCVit impl at https://github.com/NVlabs/GCVit.
The license for this code release is Apache 2.0 with no commercial restrictions.

However, weight files adapted from NVIDIA GCVit impl ARE under a non-commercial share-alike license
(https://creativecommons.org/licenses/by-nc-sa/4.0/) until I have a chance to train new ones...

Hacked together by / Copyright 2022, Ross Wightman
é    N)Úpartial)ÚCallableÚListÚOptionalÚTupleÚUnion©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚ	to_2tupleÚ	to_ntupleÚMlpÚClassifierHeadÚLayerNorm2dÚget_attnÚget_act_layerÚget_norm_layerÚ
RelPosBiasÚ_asserté   )Úbuild_model_with_cfg)Úregister_notrace_function)Únamed_apply)Úregister_modelÚgenerate_default_cfgsÚGlobalContextVitc                       s6   e Zd ZdZddddejf‡ fdd„	Zdd	„ Z‡  ZS )
ÚMbConvBlockzR A depthwise separable / fused mbconv style residual block with SE, `no norm.
    Ng      ð?ÚseFc           	   	      s¦   t ƒ  ¡  t|d�}t|tƒr&|dks.|dkr>d|d< d|d< t|ƒ}|pL|}t|| ƒ}tj||dd	d	||d
�| _	|ƒ | _
||f|Ž| _tj||d	d	d|d�| _d S )N©Ú	act_layerr   Zecag      Ð?Zrd_ratioFÚbiasé   r   )Úgroupsr!   r   ©r!   )ÚsuperÚ__init__ÚdictÚ
isinstanceÚstrr   ÚintÚnnÚConv2dÚconv_dwÚactr   Úconv_pw)	ÚselfÚin_chsÚout_chsZexpand_ratioÚ
attn_layerr!   r    Zattn_kwargsZmid_chs©Ú	__class__© úP/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/gcvit.pyr&   ,   s    	

zMbConvBlock.__init__c                 C   s8   |}|   |¡}|  |¡}|  |¡}|  |¡}|| }|S ©N)r-   r.   r   r/   )r0   ÚxZshortcutr6   r6   r7   ÚforwardC   s    



zMbConvBlock.forward)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r+   ÚGELUr&   r:   Ú__classcell__r6   r6   r4   r7   r   )   s   ùr   c                       s0   e Zd Zddejef‡ fdd„	Zdd„ Z‡  ZS )ÚDownsample2dNÚconvc                    sÈ   t ƒ  ¡  |p|}|d k	r"||ƒnt ¡ | _t||d�| _|dksFt‚|dkrhtj||ddddd�| _	nB|d	kr�||ks|t‚tj
dddd
�| _	n||ksœt‚tjdd�| _	|d k	rº||ƒnt ¡ | _d S )Nr   )rB   ÚmaxÚavgrB   r"   é   r   Fr$   rC   ©Úkernel_sizeZstrideÚpadding©rG   )r%   r&   r+   ÚIdentityÚnorm1r   Ú
conv_blockÚAssertionErrorr,   Ú	reductionÚ	MaxPool2dÚ	AvgPool2dÚnorm2)r0   ÚdimÚdim_outrN   r    Ú
norm_layerr4   r6   r7   r&   N   s    
zDownsample2d.__init__c                 C   s,   |   |¡}|  |¡}|  |¡}|  |¡}|S r8   )rK   rL   rN   rQ   ©r0   r9   r6   r6   r7   r:   f   s
    



zDownsample2d.forward)	r;   r<   r=   r+   r?   r   r&   r:   r@   r6   r6   r4   r7   rA   M   s   úrA   c                       s.   e Zd Zddejf‡ fdd„	Zdd„ Z‡  ZS )ÚFeatureBlockr   rC   c                    s¤   t ƒ  ¡  |}td|ƒ}|dkr0ttjdd�}nttjdddd�}t ¡ | _t	|ƒD ]J}| j 
d|d › �t||d�¡ |rT| j 
d	|d › �|ƒ ¡ |d8 }qTd S )
Nr   rD   rE   rI   r"   rF   rB   r   Úpool)r%   r&   rC   r   r+   rP   rO   Ú
SequentialÚblocksÚrangeZ
add_moduler   )r0   rR   ZlevelsrN   r    Z
reductionsZpool_fnÚir4   r6   r7   r&   o   s    


 zFeatureBlock.__init__c                 C   s
   |   |¡S r8   )rY   rU   r6   r6   r7   r:   „   s    zFeatureBlock.forward)r;   r<   r=   r+   r?   r&   r:   r@   r6   r6   r4   r7   rV   n   s
   ûrV   c                       s<   e Zd Zddejefeeeedœ‡ fdd„Zdd„ Z	‡  Z
S )ÚStemr"   é`   ©r1   r2   r    rT   c                    s4   t ƒ  ¡  tj||dddd�| _t|||d�| _d S )Nr"   rE   r   rF   )r    rT   )r%   r&   r+   r,   Úconv1rA   Údown)r0   r1   r2   r    rT   r4   r6   r7   r&   ‰   s    
zStem.__init__c                 C   s   |   |¡}|  |¡}|S r8   )r_   r`   rU   r6   r6   r7   r:   ”   s    

zStem.forward)r;   r<   r=   r+   r?   r   r*   r   r&   r:   r@   r6   r6   r4   r7   r\   ˆ   s   ûûr\   c                	       sN   e Zd Zd
eeeeef eeeedœ‡ fdd„Zdee	j
 dœdd	„Z‡  ZS )ÚWindowAttentionGlobalTç        )rR   Ú	num_headsÚwindow_sizeÚ
use_globalÚqkv_biasÚ	attn_dropÚ	proj_dropc                    s¦   t ƒ  ¡  t|ƒ}|| _|| _|| | _| jd | _|| _t||d�| _	| jrft
j||d |d�| _nt
j||d |d�| _t
 |¡| _t
 ||¡| _t
 |¡| _d S )Ng      à¿)rd   rc   rE   r$   r"   )r%   r&   r   rd   rc   Úhead_dimÚscalere   r   Úrel_posr+   ÚLinearÚqkvZDropoutrg   Úprojrh   )r0   rR   rc   rd   re   rf   rg   rh   r4   r6   r7   r&   œ   s    


zWindowAttentionGlobal.__init__N©Úq_globalc                 C   sV  |j \}}}| jr¬|d k	r¬t|j d |j d kdƒ |  |¡}| ||d| j| j¡ ddddd¡}| d¡\}}| 	||j d  ddd¡}	|	 ||| j| j¡ dddd¡}	n:|  |¡ ||d| j| j¡ ddddd¡}
|
 d¡\}	}}|	| j
 }	|	| dd¡ ¡  }|  |¡}|jdd	�}|  |¡}||  dd¡ |||¡}|  |¡}|  |¡}|S )
Néÿÿÿÿz*x and q_global seq lengths should be equalrE   r   r"   r   é   éþÿÿÿ)rR   )Úshapere   r   rm   Zreshaperc   ri   ÚpermuteZunbindÚrepeatrj   Z	transposeÚ
contiguousrk   Zsoftmaxrg   rn   rh   )r0   r9   rp   ÚBÚNÚCÚkvÚkÚvÚqrm   Úattnr6   r6   r7   r:   ·   s&    
$"*




zWindowAttentionGlobal.forward)TTrb   rb   )N)r;   r<   r=   r*   r   ÚboolÚfloatr&   r   ÚtorchÚTensorr:   r@   r6   r6   r4   r7   ra   š   s       ø
øra   )rd   c                 C   sj   | j \}}}}|  |||d  |d ||d  |d |¡} |  dddddd¡ ¡  d|d |d |¡}|S )Nr   r   r"   rE   rr   é   rq   ©rt   Úviewru   rw   )r9   rd   rx   ÚHÚWrz   Úwindowsr6   r6   r7   Úwindow_partitionÒ   s    ,,rŠ   )rd   Úimg_sizec                 C   sf   |\}}| j d }|  d||d  ||d  |d |d |¡}| dddddd¡ ¡  d|||¡}|S )Nrq   r   r   r"   rE   rr   r„   r…   )r‰   rd   r‹   r‡   rˆ   rz   r9   r6   r6   r7   Úwindow_reverseÙ   s
    
,$rŒ   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )Ú
LayerScaleçñhãˆµøä>Fc                    s*   t ƒ  ¡  || _t |t |¡ ¡| _d S r8   )r%   r&   Úinplacer+   Ú	Parameterr‚   ZonesÚgamma)r0   rR   Zinit_valuesr�   r4   r6   r7   r&   ã   s    
zLayerScale.__init__c                 C   s   | j r| | j¡S || j S r8   )r�   Zmul_r‘   rU   r6   r6   r7   r:   è   s    zLayerScale.forward)rŽ   F)r;   r<   r=   r&   r:   r@   r6   r6   r4   r7   r�   â   s   r�   c                       s�   e Zd Zddddddddeejejfeeeef eee	e
e
ee	 e	e	e	eeedœ‡ fdd„Zdeej d	œd
d„Zdeej d	œdd„Z‡  ZS )ÚGlobalContextVitBlocké   ç      @TNrb   )rR   Ú	feat_sizerc   rd   Ú	mlp_ratiore   rf   Úlayer_scalerh   rg   Ú	drop_pathr3   r    rT   c              	      sü   t ƒ  ¡  t|ƒ}t|ƒ}|| _t|d |d  |d |d   ƒ| _||ƒ| _|||||||
|	d�| _|d k	rzt||ƒnt	 
¡ | _|dkr”t|ƒnt	 
¡ | _||ƒ| _t|t|| ƒ||	d�| _|d k	rÔt||ƒnt	 
¡ | _|dkrît|ƒnt	 
¡ | _d S )Nr   r   )rc   rd   re   rf   rg   rh   rb   )Zin_featuresZhidden_featuresr    Zdrop)r%   r&   r   rd   r*   Znum_windowsrK   r   r�   r+   rJ   Úls1r   Ú
drop_path1rQ   r   ÚmlpÚls2Ú
drop_path2)r0   rR   r•   rc   rd   r–   re   rf   r—   rh   rg   r˜   r3   r    rT   r4   r6   r7   r&   í   s*    
&
ù	
zGlobalContextVitBlock.__init__ro   c           	      C   sZ   |j \}}}}t|| jƒ}| d| jd | jd  |¡}|  ||¡}t|| j||fƒ}|S )Nrq   r   r   )rt   rŠ   rd   r†   r   rŒ   )	r0   r9   rp   rx   r‡   rˆ   rz   Zx_winZattn_winr6   r6   r7   Ú_window_attn  s    z"GlobalContextVitBlock._window_attnc              
   C   sF   ||   |  |  |  |¡|¡¡¡ }||  |  |  |  |¡¡¡¡ }|S r8   )rš   r™   rž   rK   r�   rœ   r›   rQ   )r0   r9   rp   r6   r6   r7   r:     s    " zGlobalContextVitBlock.forward)N)N)r;   r<   r=   ra   r+   r?   Ú	LayerNormr*   r   r�   r€   r   r   r&   r‚   rƒ   rž   r:   r@   r6   r6   r4   r7   r’   ì   s:   ñ
ñ)r’   c                       s†   e Zd Zdddddddddejejefeeeeef eeef e	e	e	e
e	ee
 e
e
eee
 e
f eeedœ‡ fdd„Zd	d
„ Z‡  ZS )ÚGlobalContextVitStageTFr”   Nrb   )Údepthrc   r•   rd   Ú
downsampleÚglobal_normÚ
stage_normr–   rf   r—   rh   rg   r˜   r    rT   Únorm_layer_clc                    sø   t ƒ  ¡  |rDtˆˆd |d�| _ˆd ‰ˆd d ˆd d f‰n
t ¡ | _ˆ| _tˆƒ‰tt	 
tˆƒtˆƒ ¡ƒ}tˆ|ƒ| _|rŽˆˆƒnt ¡ | _t ‡ ‡‡‡‡‡‡‡‡‡	‡
‡fdd„t|ƒD ƒ¡| _|rØˆˆƒnt ¡ | _ˆ| _ˆ| _d| _d S )NrE   )rR   rS   rT   r   r   c                    sH   g | ]@}t ˆˆˆˆˆˆ
|d  dkˆˆ	ˆtˆtƒr8ˆ| nˆˆ ˆd�‘qS )rE   r   )rR   rc   r•   rd   r–   rf   re   r—   rh   rg   r˜   r    rT   )r’   r(   Úlist)Ú.0r[   ©r    rg   rR   r˜   r•   r—   r–   r¥   rc   rh   rf   rd   r6   r7   Ú
<listcomp>K  s    ñ
óz2GlobalContextVitStage.__init__.<locals>.<listcomp>F)r%   r&   rA   r¢   r+   rJ   r•   r   r*   ÚmathÚlog2ÚminrV   Úglobal_blockr£   Z
ModuleListrZ   rY   ÚnormrR   Úgrad_checkpointing)r0   rR   r¡   rc   r•   rd   r¢   r£   r¤   r–   rf   r—   rh   rg   r˜   r    rT   r¥   Zfeat_levelsr4   r¨   r7   r&   %  s,    
ý
$ð
zGlobalContextVitStage.__init__c                 C   s�   |   |¡}|  |¡}| dddd¡}|  | dddd¡¡}| jD ],}| jrbtj ¡ sbt	 	||¡}q@|||ƒ}q@|  
|¡}| dddd¡ ¡ }|S )Nr   rE   r"   r   )r¢   r­   ru   r£   rY   r¯   r‚   ÚjitZis_scriptingÚ
checkpointr®   rw   )r0   r9   Zglobal_queryZblkr6   r6   r7   r:   b  s    



zGlobalContextVitStage.forward)r;   r<   r=   r+   r?   rŸ   r   r*   r   r€   r�   r   r   r   r   r&   r:   r@   r6   r6   r4   r7   r    $  s>   î

î=r    c                       s  e Zd Zd,eeeeeef eedf eedf eeedf eedf eeee eeeeeeeedœ‡ fdd„Z	d-dd„Z
ejjdd„ ƒZejjd.dd„ƒZejjd/dd„ƒZejjd d!„ ƒZd0d"d#„Zejejd$œd%d&„Zd1ed'œd(d)„Zejejd$œd*d+„Z‡  ZS )2r   r"   éè  rD   éà   ©é    rµ   é   rµ   Né@   ©r"   rr   é   r„   ©rE   rr   é   r¶   ç      @Trb   Ú ÚgeluÚlayernorm2dÚ	layernormrŽ   .)Úin_chansÚnum_classesÚglobal_poolr‹   Úwindow_ratiord   Ú	embed_dimÚdepthsrc   r–   rf   r—   Ú	drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rater    rT   r¥   Únorm_epsc                    s¸  t ƒ  ¡  t|ƒ}tt|ƒ|d�}tt|ƒ|d�}tˆ ƒ‰ tdd„ ˆ D ƒƒ}|| _|| _|| _	t
|ƒ}t|d|d   ƒ| _|d k	r’t|ƒ|ƒ}n*|d k	sžt‚t‡ fdd„t|ƒ|ƒD ƒƒ}t||||d�| _d	d„ t d
|t|ƒ¡ |¡D ƒ}g }t|ƒD ]|}||d k}dt|d d
ƒ }| t|| || |	| |d
 | |d | f|| |d
k||
|||||| |||d�¡ qütj|Ž | _t| j|||d�| _|�r´tt| j|d�| ƒ d S )N)Úepsc                 s   s   | ]}|d  V  qdS )rr   Nr6   )r§   Údr6   r6   r7   Ú	<genexpr>“  s     z,GlobalContextVit.__init__.<locals>.<genexpr>rE   r   c                    s$   g | ]}ˆ d  | ˆ d | f‘qS )r   r   r6   )r§   Úr©r‹   r6   r7   r©   �  s     z-GlobalContextVit.__init__.<locals>.<listcomp>r^   c                 S   s   g | ]}|  ¡ ‘qS r6   )Útolist)r§   r9   r6   r6   r7   r©   ¦  s     r   )rR   r¡   rc   r•   rd   r¢   r¤   r–   rf   r—   rh   rg   r˜   r    rT   r¥   ©Ú	pool_typerÇ   )Úscheme) r%   r&   r   r   r   r   ÚtuplerÃ   rÂ   rÇ   Úlenr*   Únum_featuresr   rM   r\   Ústemr‚   ZlinspaceÚsumÚsplitrZ   rC   Úappendr    r+   rX   Ústagesr   Úheadr   Ú_init_weights)r0   rÁ   rÂ   rÃ   r‹   rÄ   rd   rÅ   rÆ   rc   r–   rf   r—   rÇ   rÈ   rÉ   rÊ   Zweight_initr    rT   r¥   rË   r•   Z
num_stagesZdprrÜ   r[   Z
last_stageZstage_scaler4   rÐ   r7   r&   u  s`    
ü"ð
zGlobalContextVit.__init__Úvitc                 C   s’   |dkrXt |tjƒrŽtj |j¡ |jd k	rŽd|krHtjj|jdd� qŽtj |j¡ n6t |tjƒrŽtjj|jdd� |jd k	rŽtj |j¡ d S )Nrß   r›   g�íµ ÷Æ°>)Ústdg{®Gáz”?)	r(   r+   rl   ÚinitZxavier_uniform_Úweightr!   Znormal_Zzeros_)r0   ÚmoduleÚnamerÔ   r6   r6   r7   rÞ   Å  s    

zGlobalContextVit._init_weightsc                 C   s   dd„ |   ¡ D ƒS )Nc                    s*   h | ]"\‰ }t ‡ fd d„dD ƒƒrˆ ’qS )c                 3   s   | ]}|ˆ kV  qd S r8   r6   )r§   Ún©r|   r6   r7   rÎ   Ù  s     z=GlobalContextVit.no_weight_decay.<locals>.<setcomp>.<genexpr>)Zrelative_position_bias_tablezrel_pos.mlp)Úany)r§   Ú_r6   ræ   r7   Ú	<setcomp>×  s   ÿz3GlobalContextVit.no_weight_decay.<locals>.<setcomp>)Znamed_parameters©r0   r6   r6   r7   Úno_weight_decayÕ  s    ÿz GlobalContextVit.no_weight_decayFc                 C   s   t ddd�}|S )Nz^stemz^stages\.(\d+))rØ   rY   )r'   )r0   ZcoarseZmatcherr6   r6   r7   Úgroup_matcherÛ  s
    þzGlobalContextVit.group_matcherc                 C   s   | j D ]
}||_qd S r8   )rÜ   r¯   )r0   ÚenableÚsr6   r6   r7   Úset_grad_checkpointingã  s    
z'GlobalContextVit.set_grad_checkpointingc                 C   s   | j jS r8   )rÝ   Úfcrê   r6   r6   r7   Úget_classifierè  s    zGlobalContextVit.get_classifierc                 C   s2   || _ |d kr| jjj}t| j||| jd�| _d S )NrÒ   )rÂ   rÝ   rÃ   rÓ   r   r×   rÇ   )r0   rÂ   rÃ   r6   r6   r7   Úreset_classifierì  s    
z!GlobalContextVit.reset_classifier)r9   Úreturnc                 C   s   |   |¡}|  |¡}|S r8   )rØ   rÜ   rU   r6   r6   r7   Úforward_featuresò  s    

z!GlobalContextVit.forward_features©Ú
pre_logitsc                 C   s   | j ||d�S )Nrõ   )rÝ   )r0   r9   rö   r6   r6   r7   Úforward_head÷  s    zGlobalContextVit.forward_headc                 C   s   |   |¡}|  |¡}|S r8   )rô   r÷   rU   r6   r6   r7   r:   ú  s    

zGlobalContextVit.forward)r"   r²   rD   r³   r´   Nr·   r¸   rº   r¼   TNrb   rb   rb   rb   r½   r¾   r¿   rÀ   rŽ   )rß   )F)T)N)F)r;   r<   r=   r*   r)   r   r�   r€   r   r&   rÞ   r‚   r°   Úignorerë   rì   rï   rñ   rò   rƒ   rô   r÷   r:   r@   r6   r6   r4   r7   r   t  sp                        ê




êP



Fc                 K   s(   |  dd ¡rtdƒ‚tt| |f|Ž}|S )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.)ÚgetÚRuntimeErrorr   r   )ÚvariantÚ
pretrainedÚkwargsÚmodelr6   r6   r7   Ú_create_gcvit   s    rÿ   r½   c                 K   s    | dddddt tdddd	œ|–S )
Nr²   )r"   r³   r³   )r“   r“   g      ì?Zbicubicz
stem.conv1zhead.fcT)ÚurlrÂ   Z
input_sizeZ	pool_sizeZcrop_pctÚinterpolationZmeanrà   Z
first_convÚ
classifierZfixed_input_sizer   )r   rý   r6   r6   r7   Ú_cfg  s          ûúr  z}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xxtiny_224_nvidia-d1d86009.pth)r   z|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xtiny_224_nvidia-274b92b7.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_tiny_224_nvidia-ac783954.pthz|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_small_224_nvidia-4e98afa2.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_base_224_nvidia-f009139b.pth)zgcvit_xxtiny.in1kzgcvit_xtiny.in1kzgcvit_tiny.in1kzgcvit_small.in1kzgcvit_base.in1k)ró   c                 K   s&   t f dddœ|—Ž}tdd| i|—ŽS )N)rE   rE   é   rE   rº   ©rÆ   rc   Úgcvit_xxtinyrü   )r  ©r'   rÿ   ©rü   rý   Zmodel_kwargsr6   r6   r7   r     s    þýr  c                 K   s&   t f dddœ|—Ž}tdd| i|—ŽS )N)r"   rr   r  r„   rº   r  Úgcvit_xtinyrü   )r	  r  r  r6   r6   r7   r	  )  s    þýr	  c                 K   s&   t f dddœ|—Ž}tdd| i|—ŽS )Nr¸   rº   r  Ú
gcvit_tinyrü   )r
  r  r  r6   r6   r7   r
  2  s    þýr
  c                 K   s,   t f ddddddœ|—Ž}td	d| i|—ŽS )
Nr¸   )r"   r  é   é   r]   rE   rŽ   ©rÆ   rc   rÅ   r–   r—   Úgcvit_smallrü   )r  r  r  r6   r6   r7   r  ;  s    ûúr  c                 K   s,   t f ddddddœ|—Ž}td	d| i|—ŽS )
Nr¸   )rr   r»   r¶   rµ   é€   rE   rŽ   r  Ú
gcvit_baserü   )r  r  r  r6   r6   r7   r  G  s    ûúr  )F)r½   )F)F)F)F)F)>r>   rª   Ú	functoolsr   Útypingr   r   r   r   r   r‚   Ztorch.nnr+   Ztorch.utils.checkpointÚutilsr±   Z	timm.datar	   r
   Ztimm.layersr   r   r   r   r   r   r   r   r   r   r   Z_builderr   Z_features_fxr   Z_manipulater   Ú	_registryr   r   Ú__all__ÚModuler   rA   rV   r\   ra   r*   rŠ   rŒ   r�   r’   r    r   rÿ   r  Zdefault_cfgsr  r	  r
  r  r  r6   r6   r6   r7   Ú<module>   sn   4$!8"
8P 

ÿÿÿÿÿ÷