U
    ¯Ê¹jõK ã                K   @   sà  d Z ddlZddlmZ ddlmZmZmZ ddlm	Z	 ddl
mZmZmZmZmZ ddlZddlmZ ddlmZ dd	lmZmZ dd
lmZmZmZmZmZmZ ddlmZmZm Z m!Z!m"Z"m#Z# ddlm$Z$m%Z%m&Z&m'Z'm(Z( ddlm)Z)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6m7Z7 ddddgZ8eG dd„ dƒƒZ9eG dd„ dƒƒZ:eG dd„ dƒƒZ;G dd„ dej<ƒZ=G dd„ dej<ƒZ>G dd„ dej<ƒZ?G d d!„ d!ej<ƒZ@G d"d#„ d#ej<ƒZA�dUd%d&„ZBG d'd(„ d(ej<ƒZC�dVd)d*„ZDd+d,„ ZEG d-d.„ d.ej<ƒZFG d/d0„ d0ej<ƒZGeeH d1œd2d3„ZIe1eeH eeH d4œd5d6„ƒZJeeH d7œd8d9„ZKe1eeH eeH d:œd;d<„ƒZLe9d=œd>d?„ZMG d@dA„ dAej<ƒZNG dBdC„ dCej<ƒZOeeH d1œdDdE„ZPe1eeH eeH d4œdFdG„ƒZQeeH d7œdHdI„ZRe1eeH eeH d:œdJdK„ƒZSG dLdM„ dMej<ƒZTG dNdO„ dOej<ƒZUG dPdQ„ dQej<ƒZVG dRdS„ dSej<ƒZWG dTdU„ dUej<ƒZXe9eeHeHf dVœdWdX„ZYe;d=œdYdZ„ZZG d[d„ dej<ƒZ[�dWdedf„Z\�dXdjdk„Z]�dYdndo„Z^dpdq„ Z_e`e;f drdsdtduœe]d`dvdw�—Že;f drdxdtduœe]d\d`dvdy�—Že;f dzd{dtduœe\d`d^d|�—Že;f dzd}dtduœe\dgd`d^d~�—Že;f dd}d€duœe\dgd�d‚�—Že;f dƒd}d„duœe\dgd�dld…�—Že;f dzd{dtduœe\dgd`d^d†d‡�—Že;f drdxdtduœe]d`dvdmdˆd‰�—Že;f dzd{dtduœe\dgdmdŠ�—Že;f dzd}dtduœe\d‹d`d^dmdˆdŒ�—Že;f dzd}dtduœe\dgdmddd��—Že;f dd}d€duœe\dgd�dldmdŽ�—Že;f dƒd}d„duœe\dgd�dldmdŽ�—Že;f drdxdtd�d�œe\ƒ —Že;f drdxdtd‘d’œe^dcd“d”�—Že;dzdsd•d–d—�e;dzd}d•d–d—�e;dd}d˜d™d—�e;dƒd}dšd›d—�e;dƒdœdšd›d—�e;d�dœdšdžd—�e;f dŸd d¡d¢d£œe]ƒ —Že;f drd¤d¡dtd£œe]ƒ —Že;f drd d¡dtd£œe]ƒ —Že;f drd d¥dtd£œe]ƒ —Že;f dŸd d¡d¢d£œe]dmd¦�—Že;f drd¤d¡dtd£œe]dmd¦�—Že;f drd d¡dtd£œe]dmd¦�—Že;f dzd d¡dtd£œe]dmdld”�—Že;f dzd}d¡dtd–d§œe]dmd¦�—Že;f drd¤d¡dtd‘d¨œe^ƒ —Že;f drd d¡dtd£œe^ƒ —Že;f dzd d¡d©d£œe^ƒ —Že;f dzd¤d¡d©d‘d¨œe^d`dcdª�—Že;f dd«d¡d€d£œe^d`d¬�—Že;f d­d®d¡d¯d°d§œe^d`d¬�—Že;f drd d¡d•d`ddd±œe_ƒ —Že;f dzd d¡d•d`d–d±œe_ƒ —Že;f dzd}d¡d•d`d–d±œe_ƒ —Že;f dd}d¡d˜d`d™d±œe_ƒ —Že;f dƒd}d¡dšd`d›d±œe_ƒ —Žd²�)Zaej<d³œd´dµ„Zb�dZd¶d·„Zc�d[d¸d¹„Zde6edd$dº�edd»d¼d½d¾�edd»d¿dÀ�edd»dÁdÀ�edd»dÂ�edd»dÂ�edd»dÂ�edd»dÃdÄdÅdÆdÇ�edd»dÈeedÉdÊ�edd»dËd½d¾�edd$dº�edd»dÌdÀ�edd»dÍdÀ�edd$dº�edd$dº�edd»dÎd½d¾�edd»dÏdÐ�edd»dÏdÐ�edd»dÏdÐ�edd»dÏdÐ�edd$dº�edd$dº�edd$dº�edd$dº�edd$dº�edd$dº�edd$dÑdÒdÓ�edd»dÔdÑdÒdÕ�edd»dÖdÀ�edd$dÑdÒdÓ�edd$dÑdÒdÓ�edd»d×dÑdÒdÕ�edd»dØdÑdÒdÕ�edd»dÙdÑdÒdÕ�edd»dÚd½d¾�edd$dÑdÒdÓ�edd»dÂ�edd»dÃdÄdÅdÆdÇ�edd»dÏdÐ�edd»dÛdÑdÒdÕ�edd$dÑdÒdÓ�edd»dÜdÑdÒdÕ�edd»dÑdÒdÝ�edd»dÂ�edd»dÃdÄdÅdÆdÇ�edd$dº�edd»dÏdÐ�edd»eedÞ�edd»dÃdÄdÅdÆdÇ�edd»dßdàdÅdÆdÇ�edd»eedÞ�edd»dÃdÄdÅdÆdÇ�edd»dßdàdÅdÆdÇ�edd»eedÞ�edd»dÃdÄdÅdÆdÇ�edd»dßdàdÅdÆdÇ�edd»eedÞ�edd»dÃdÄdÅdÆdÇ�edd»dßdàdÅdÆdÇ�edd»dádÐ�edd»dÃdÄdÅdÆdÇ�edd»dßdàdÅdÆdÇ�edd»dádÐ�edd»dÃdÄdÅdÆdÇ�edd»dßdÅdÆdâ�edd»dádÐ�edd»dÃdÄdÅdÆdÇ�edd»dßdàdÅdÆdÇ�dãœDƒZee7�d\e[däœdådæ„ƒZfe7�d]e[däœdçdè„ƒZge7�d^e[däœdédê„ƒZhe7�d_e[däœdëdì„ƒZie7�d`e[däœdídî„ƒZje7�dae[däœdïdð„ƒZke7�dbe[däœdñdò„ƒZle7�dce[däœdódô„ƒZme7�dde[däœdõdö„ƒZne7�dee[däœd÷dø„ƒZoe7�dfe[däœdùdú„ƒZpe7�dge[däœdûdü„ƒZqe7�dhe[däœdýdþ„ƒZre7�die[däœdÿ�d „ƒZse7�dje[däœ�d�d„ƒZte7�dke[däœ�d�d„ƒZue7�dle[däœ�d�d„ƒZve7�dme[däœ�d�d„ƒZwe7�dne[däœ�d	�d
„ƒZxe7�doe[däœ�d�d„ƒZye7�dpe[däœ�d�d„ƒZze7�dqe[däœ�d�d„ƒZ{e7�dre[däœ�d�d„ƒZ|e7�dse[däœ�d�d„ƒZ}e7�dte[däœ�d�d„ƒZ~e7�due[däœ�d�d„ƒZe7�dve[däœ�d�d„ƒZ€e7�dwe[däœ�d�d„ƒZ�e7�dxe[däœ�d�d„ƒZ‚e7�dye[däœ�d�d „ƒZƒe7�dze[däœ�d!�d"„ƒZ„e7�d{e[däœ�d#�d$„ƒZ…e7�d|e[däœ�d%�d&„ƒZ†e7�d}e[däœ�d'�d(„ƒZ‡e7�d~e[däœ�d)�d*„ƒZˆe7�de[däœ�d+�d,„ƒZ‰e7�d€e[däœ�d-�d.„ƒZŠe7�d�e[däœ�d/�d0„ƒZ‹e7�d‚e[däœ�d1�d2„ƒZŒe7�dƒe[däœ�d3�d4„ƒZ�e7�d„e[däœ�d5�d6„ƒZŽe7�d…e[däœ�d7�d8„ƒZ�e7�d†e[däœ�d9�d:„ƒZ�e7�d‡e[däœ�d;�d<„ƒZ‘e7�dˆe[däœ�d=�d>„ƒZ’e7�d‰e[däœ�d?�d@„ƒZ“e7�dŠe[däœ�dA�dB„ƒZ”e7�d‹e[däœ�dC�dD„ƒZ•e7�dŒe[däœ�dE�dF„ƒZ–e7�d�e[däœ�dG�dH„ƒZ—e7�dŽe[däœ�dI�dJ„ƒZ˜e7�d�e[däœ�dK�dL„ƒZ™e7�d�e[däœ�dM�dN„ƒZše7�d‘e[däœ�dO�dP„ƒZ›e7�d’e[däœ�dQ�dR„ƒZœe7�d“e[däœ�dS�dT„ƒZ�dS (”  a   MaxVit and CoAtNet Vision Transformer - CNN Hybrids in PyTorch

This is a from-scratch implementation of both CoAtNet and MaxVit in PyTorch.

99% of the implementation was done from papers, however last minute some adjustments were made
based on the (as yet unfinished?) public code release https://github.com/google-research/maxvit

There are multiple sets of models defined for both architectures. Typically, names with a
 `_rw` suffix are my own original configs prior to referencing https://github.com/google-research/maxvit.
These configs work well and appear to be a bit faster / lower resource than the paper.

The models without extra prefix / suffix' (coatnet_0_224, maxvit_tiny_224, etc), are intended to
match paper, BUT, without any official pretrained weights it's difficult to confirm a 100% match.

Papers:

MaxViT: Multi-Axis Vision Transformer - https://arxiv.org/abs/2204.01697
@article{tu2022maxvit,
  title={MaxViT: Multi-Axis Vision Transformer},
  author={Tu, Zhengzhong and Talebi, Hossein and Zhang, Han and Yang, Feng and Milanfar, Peyman and Bovik, Alan and Li, Yinxiao},
  journal={ECCV},
  year={2022},
}

CoAtNet: Marrying Convolution and Attention for All Data Sizes - https://arxiv.org/abs/2106.04803
@article{DBLP:journals/corr/abs-2106-04803,
  author    = {Zihang Dai and Hanxiao Liu and Quoc V. Le and Mingxing Tan},
  title     = {CoAtNet: Marrying Convolution and Attention for All Data Sizes},
  journal   = {CoRR},
  volume    = {abs/2106.04803},
  year      = {2021}
}

Hacked together by / Copyright 2022, Ross Wightman
é    N)ÚOrderedDict)Ú	dataclassÚreplaceÚfield)Úpartial)ÚCallableÚOptionalÚUnionÚTupleÚList)Únn)ÚFinal)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚMlpÚConvMlpÚDropPathÚ	LayerNormÚClassifierHeadÚNormMlpClassifierHead)Úcreate_attnÚget_act_layerÚget_norm_layerÚget_norm_act_layerÚcreate_conv2dÚcreate_pool2d)Útrunc_normal_tf_Ú	to_2tupleÚextend_tupleÚmake_divisibleÚ_assert)Ú	RelPosMlpÚ
RelPosBiasÚRelPosBiasTfÚuse_fused_attnÚresize_rel_pos_bias_tableé   )Úbuild_model_with_cfg)Úregister_notrace_function)Únamed_applyÚcheckpoint_seq)Úgenerate_default_cfgsÚregister_modelÚ
MaxxVitCfgÚMaxxVitConvCfgÚMaxxVitTransformerCfgÚMaxxVitc                   @   s.  e Zd ZU dZeed< dZeed< dZe	ed< dZ
eed< dZeed< dZeed	< d
Ze	ed< d
Ze	ed< dZeed< dZeed< dZeed< dZeed< dZeeeef  ed< dZeeeef  ed< dZeed< dZeed< dZee	 ed< dZeed< dZeed< dZeed < d!Ze	ed"< d#d$„ ZdS )%r/   é    Údim_headTÚ
head_firstç      @Úexpand_ratioÚexpand_firstÚshortcut_biasÚ	attn_biasç        Ú	attn_dropÚ	proj_dropÚavg2Ú	pool_typeÚbiasÚrel_pos_typeé   Úrel_pos_dimÚpartition_ratioNÚwindow_sizeÚ	grid_sizeFÚno_block_attnÚuse_nchw_attnÚinit_valuesÚgeluÚ	act_layerÚlayernorm2dÚ
norm_layerÚ	layernormÚnorm_layer_clç�íµ ÷Æ°>Únorm_epsc                 C   sB   | j d k	rt| j ƒ| _ | jd k	r>t| jƒ| _| j d kr>| j| _ d S ©N)rD   r   rC   ©Úself© rS   úR/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/maxxvit.pyÚ__post_init__T   s    


z#MaxxVitTransformerCfg.__post_init__) Ú__name__Ú
__module__Ú__qualname__r2   ÚintÚ__annotations__r3   Úboolr5   Úfloatr6   r7   r8   r:   r;   r=   Ústrr?   rA   rB   rC   r   r
   rD   rE   rF   rG   rI   rK   rM   rO   rU   rS   rS   rS   rT   r/   <   s,   
c                   @   s  e Zd ZU dZeed< dZeed< dZe	ed< dZ
eed< d	Zeed
< dZe	ed< dZe	ed< dZeed< dZeed< dZeed< dZeed< dZe	ed< dZeed< dZeed< dZeed< dZee ed< dZeed< dZeed < dZeed!< d"Zee ed#< d$d%„ Zd"S )&r.   ÚmbconvÚ
block_typer4   r5   TÚexpand_outputé   Úkernel_sizer&   Ú
group_sizeFÚpre_norm_actÚoutput_biasÚdwÚstride_moder<   r=   Údownsample_pool_typeÚ ÚpaddingÚ
attn_earlyÚseÚ
attn_layerÚsiluÚattn_act_layerç      Ð?Ú
attn_ratiorN   rG   rH   rI   rK   rM   NrO   c                 C   sf   | j dkst‚| j dk}| js,|r&dnd| _| js<|s<d| _| jd krT|rNdnd| _| jp^| j| _d S )N)r^   Úconvnextr^   Úbatchnorm2drJ   rL   çñhãˆµøä>rN   )r_   ÚAssertionErrorrK   rM   rO   rh   r=   )rR   Z
use_mbconvrS   rS   rT   rU   t   s    


zMaxxVitConvCfg.__post_init__)rV   rW   rX   r_   r]   rZ   r5   r\   r`   r[   rb   rY   rc   rd   re   rg   r=   rh   rj   rk   rm   ro   rq   rG   r   rI   rK   rM   rO   rU   rS   rS   rS   rT   r.   ]   s*   
c                   @   s¾   e Zd ZU dZeedf ed< dZeedf ed< dZee	e
ee
df f df ed< dZe	eeeef f ed	< d
Zeed< eed�Zeed< eed�Zeed< dZeed< dZe
ed< dS )r-   ©é`   éÀ   é€  é   .Ú	embed_dim©é   ra   é   r}   Údepths)ÚCr€   ÚTr�   r_   é@   Ú
stem_widthFÚ	stem_bias)Údefault_factoryÚconv_cfgÚtransformer_cfgNÚhead_hidden_sizeÚvit_effÚweight_init)rV   rW   rX   r{   r
   rY   rZ   r   r_   r	   r]   rƒ   r„   r[   r   r.   r†   r/   r‡   rˆ   rŠ   rS   rS   rS   rT   r-   �   s   
$c                       s\   e Zd ZU ee ed< deee eeeeee	e	dœ	‡ fdd„Z
deej d	œd
d„Z‡  ZS )ÚAttention2dÚ
fused_attnNr1   Tr9   ©	ÚdimÚdim_outr2   r>   r6   r3   Úrel_pos_clsr:   r;   c
                    s¤   t ƒ  ¡  |p|}|r|n|}
|
| | _|| _|| _|d | _tƒ | _tj	||
d d|d�| _
|rn|| jd�nd | _t |¡| _tj	|
|d|d�| _t |	¡| _d S )Nç      à¿ra   r&   ©r>   ©Ú	num_heads)ÚsuperÚ__init__r”   r2   r3   Úscaler$   rŒ   r   ÚConv2dÚqkvÚrel_posÚDropoutr:   Úprojr;   ©rR   rŽ   r�   r2   r>   r6   r3   r�   r:   r;   Zdim_attn©Ú	__class__rS   rT   r–   ’   s    


zAttention2d.__init__©Úshared_rel_posc                 C   s†  |j \}}}}| jrD|  |¡ || j| jd d¡jddd�\}}}	n(|  |¡ |d| j| jd¡ d¡\}}}	| j	rød }
| j
d k	rŒ| j
 ¡ }
n|d k	r˜|}
tjjj| dd¡ ¡ | dd¡ ¡ |	 dd¡ ¡ |
| jrÚ| jjndd� dd¡ |d||¡}nv|| j }| dd¡| }| j
d k	�r*|  
|¡}n|d k	�r<|| }|jdd�}|  |¡}|	| dd¡  |d||¡}|  |¡}|  |¡}|S )	Nra   éÿÿÿÿr}   ©rŽ   r&   éþÿÿÿr9   ©Z	attn_maskZ	dropout_p)Úshaper3   r™   Úviewr”   r2   ÚchunkÚreshapeÚunbindrŒ   rš   Úget_biasÚtorchr   Ú
functionalÚscaled_dot_product_attentionÚ	transposeÚ
contiguousÚtrainingr:   Úpr—   Úsoftmaxrœ   r;   )rR   Úxr¡   ÚBr€   ÚHÚWÚqÚkÚvr8   ÚattnrS   rS   rT   Úforward­   sJ    0(
û ú   ú




zAttention2d.forward)Nr1   TTTNr9   r9   )N)rV   rW   rX   r   r[   rZ   rY   r   r   r\   r–   r¬   ÚTensorr¼   Ú__classcell__rS   rS   rž   rT   r‹   Ž   s*   
        öör‹   c                       s`   e Zd ZU dZee ed< deee eeeee	e
e
dœ	‡ fdd	„Zdeej d
œdd„Z‡  ZS )ÚAttentionClz0 Channels-last multi-head attention (B, ..., C) rŒ   Nr1   Tr9   r�   c
                    s¼   t ƒ  ¡  |p|}|r"||kr"|n|}
|
| dks:tdƒ‚|
| | _|| _|| _|d | _tƒ | _t	j
||
d |d�| _|rˆ|| jd�nd | _t	 |¡| _t	j
|
||d�| _t	 |	¡| _d S )Nr   z(attn dim should be divisible by head_dimr‘   ra   r’   r“   )r•   r–   ru   r”   r2   r3   r—   r$   rŒ   r   ÚLinearr™   rš   r›   r:   rœ   r;   r�   rž   rS   rT   r–   ×   s    


zAttentionCl.__init__r    c           
      C   sn  |j d }|j d d… }| jrV|  |¡ |d| j| jd ¡ dd¡jddd�\}}}n0|  |¡ |dd| j| j¡ dd¡ 	d¡\}}}| j
rÚd }| jd k	r¦| j ¡ }n|d k	r²|}tjjj||||| jrÐ| jjndd�}nf|| j }|| d	d¡ }	| jd k	�r| j|	|d
�}	n|d k	�r"|	| }	|	jdd�}	|  |	¡}	|	| }| dd¡ |d ¡}|  |¡}|  |¡}|S )Nr   r¢   ra   r&   r}   r£   r9   r¥   r¤   r    )r¢   )r¦   r3   r™   r§   r”   r2   r¯   r¨   r©   rª   rŒ   rš   r«   r¬   r   r­   r®   r±   r:   r²   r—   r³   rœ   r;   )
rR   r´   r¡   rµ   Zrestore_shaper¸   r¹   rº   r8   r»   rS   rS   rT   r¼   ó   s>    
80
  ý




zAttentionCl.forward)Nr1   TTTNr9   r9   )N)rV   rW   rX   Ú__doc__r   r[   rZ   rY   r   r   r\   r–   r¬   r½   r¼   r¾   rS   rS   rž   rT   r¿   Ó   s,   
        öör¿   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )Ú
LayerScalert   Fc                    s*   t ƒ  ¡  || _t |t |¡ ¡| _d S rP   ©r•   r–   Úinplacer   Ú	Parameterr¬   ZonesÚgamma©rR   rŽ   rG   rÄ   rž   rS   rT   r–     s    
zLayerScale.__init__c                 C   s   | j }| jr| |¡S || S rP   )rÆ   rÄ   Úmul_©rR   r´   rÆ   rS   rS   rT   r¼     s    zLayerScale.forward)rt   F©rV   rW   rX   r–   r¼   r¾   rS   rS   rž   rT   rÂ     s   rÂ   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )ÚLayerScale2drt   Fc                    s*   t ƒ  ¡  || _t |t |¡ ¡| _d S rP   rÃ   rÇ   rž   rS   rT   r–   %  s    
zLayerScale2d.__init__c                 C   s*   | j  dddd¡}| jr"| |¡S || S )Nr&   r¢   )rÆ   r§   rÄ   rÈ   rÉ   rS   rS   rT   r¼   *  s    zLayerScale2d.forward)rt   FrÊ   rS   rS   rž   rT   rË   $  s   rË   c                       s8   e Zd ZdZd
eeeeedœ‡ fdd„Zdd	„ Z‡  Z	S )ÚDownsample2da4   A downsample pooling module supporting several maxpool and avgpool modes
    * 'max' - MaxPool2d w/ kernel_size 3, stride 2, padding 1
    * 'max2' - MaxPool2d w/ kernel_size = stride = 2
    * 'avg' - AvgPool2d w/ kernel_size 3, stride 2, padding 1
    * 'avg2' - AvgPool2d w/ kernel_size = stride = 2
    r<   ri   T)rŽ   r�   r=   rj   r>   c                    s¶   t ƒ  ¡  |dkst‚|dkr6tddd|p,dd�| _nT|dkrTtdd|pJdd	�| _n6|d
krvtd
ddd|pldd�| _ntd
d|p‚dd	�| _||kr¨tj||d|d�| _n
t ¡ | _d S )N)ÚmaxÚmax2Úavgr<   rÍ   ra   r}   r&   )rb   Ústriderj   rÎ   r   )rj   rÏ   F)rb   rÐ   Zcount_include_padrj   r’   )	r•   r–   ru   r   Úpoolr   r˜   ÚexpandÚIdentity)rR   rŽ   r�   r=   rj   r>   rž   rS   rT   r–   7  s$    
    ÿ
zDownsample2d.__init__c                 C   s   |   |¡}|  |¡}|S rP   )rÑ   rÒ   ©rR   r´   rS   rS   rT   r¼   P  s    

zDownsample2d.forward)r<   ri   T)
rV   rW   rX   rÁ   rY   r]   r[   r–   r¼   r¾   rS   rS   rž   rT   rÌ   /  s      úúrÌ   ri   c                 C   sì   t | tjtjfƒrè|dkrFtjj| jdd� | jd k	rètj | j¡ n¢|dkrvt	| jdd� | jd k	rètj | j¡ nr|dkr¦tj 
| j¡ | jd k	rètj | j¡ nBtj | j¡ | jd k	rèd|krÚtjj| jdd� ntj | j¡ d S )NÚnormalç{®Gáz”?©ÚstdÚtrunc_normalÚxavier_normalÚmlprN   )Ú
isinstancer   r˜   rÀ   ÚinitÚnormal_Úweightr>   Úzeros_r   Úxavier_normal_Zxavier_uniform_)ÚmoduleÚnameÚschemerS   rS   rT   Ú_init_transformerV  s$    



rå   c                       s\   e Zd ZdZddeƒ dfeeeeeedœ‡ fdd„Zdd	d
„Z	de
ej dœdd„Z‡  ZS )ÚTransformerBlock2daX   Transformer block with 2D downsampling
    '2D' NCHW tensor layout

    Some gains can be seen on GPU using a 1D / CL block, BUT w/ the need to switch back/forth to NCHW
    for spatial pooling, the benefit is minimal so ended up using just this variant for CoAt configs.

    This impl was faster on TPU w/ PT XLA than the 1D experiment.
    r&   Nr9   )rŽ   r�   rÐ   r�   ÚcfgÚ	drop_pathc           	   
      sX  t ƒ  ¡  tt|jƒ|jd�}t|jƒ}|dkrtt|||j	|j
d�| _t td||ƒfdt|||j	d�fgƒ¡| _n ||ks€t‚t ¡ | _||ƒ| _t|||j|j|j||j|jd�| _|jrÌt||jd�nt ¡ | _|d	kræt|ƒnt ¡ | _||ƒ| _t|t ||j! ƒ||jd
�| _"|j�r.t||jd�nt ¡ | _#|d	k�rJt|ƒnt ¡ | _$d S )N©Úepsr}   )r=   r>   ÚnormÚdown©r=   )r2   r6   r>   r�   r:   r;   ©rG   r9   ©Úin_featuresÚhidden_featuresrI   Údrop)%r•   r–   r   r   rK   rO   r   rI   rÌ   r=   r7   Úshortcutr   Ú
Sequentialr   Únorm1ru   rÓ   r‹   r2   r6   r8   r:   r;   r»   rG   rË   Úls1r   Ú
drop_path1Únorm2r   rY   r5   rÛ   Úls2Ú
drop_path2)	rR   rŽ   r�   rÐ   r�   rç   rè   rK   rI   rž   rS   rT   r–   x  sB    	


þ

ø

ü zTransformerBlock2d.__init__ri   c                 C   s   t tt|d�| ƒ d S ©N©rä   )r)   r   rå   ©rR   rä   rS   rS   rT   Úinit_weights¦  s    zTransformerBlock2d.init_weightsr    c              
   C   sN   |   |¡|  |  | j|  |¡|d�¡¡ }||  |  |  |  |¡¡¡¡ }|S )Nr    )	ró   r÷   rö   r»   rõ   rú   rù   rÛ   rø   )rR   r´   r¡   rS   rS   rT   r¼   ©  s    * zTransformerBlock2d.forward)ri   )N)rV   rW   rX   rÁ   r/   rY   r   r\   r–   rþ   r   r¬   r½   r¼   r¾   rS   rS   rž   rT   ræ   n  s   ùù.
ræ   c                 C   sü   t | tjƒrø|dkr@tjj| jdd� | jd k	røtj | j¡ n¸|dkrpt| jdd� | jd k	røtj | j¡ nˆ|dkr tj 	| j¡ | jd k	røtj | j¡ nX| j
d | j
d  | j }|| j }tj | jdt d| ¡¡ | jd k	røtj | j¡ d S )	NrÕ   rÖ   r×   rÙ   rÚ   r   r&   g       @)rÜ   r   r˜   rÝ   rÞ   rß   r>   rà   r   rá   rb   Zout_channelsÚgroupsÚmathÚsqrt)râ   rã   rä   Zfan_outrS   rS   rT   Ú
_init_conv¯  s$    




r  c                 C   s$   | sdS ||  dkst ‚||  S d S )Nr&   r   )ru   )rc   ZchannelsrS   rS   rT   Ú
num_groupsÆ  s    r  c                       sV   e Zd ZdZddeƒ dfeeeeeef eedœ‡ fdd„Zdd	d
„Z	dd„ Z
‡  ZS )ÚMbConvBlockzL Pre-Norm Conv Block - 1x1 - kxk - 1x1, w/ inverted bottleneck (expand)
    r&   ©r&   r&   r9   )Úin_chsÚout_chsrÐ   Údilationrç   rè   c              	      s  t t| ƒ ¡  tt|j|jƒ|jd�}t|j	r2|n||j
 ƒ}t|j|ƒ}	|dkrnt|||j|j|jd�| _n
t ¡ | _|jdks†t‚d\}
}}|jdkrª||d  }
}n(|jdkrÄ||d  }}n||d	  }}|||jd
�| _|
dk�rt|||j|jd�| _n
t ¡ | _t||d|d�| _||ƒ| _t|||j|||	|jd�| _i }t|j t!ƒ�r˜|j dk�sp|j dk�r˜|j"|d< t#|j$|j	�rŒ|n| ƒ|d< |j%�rÄt&|j |f|Ž| _'||ƒ| _(d | _)n"d | _'||ƒ| _(t&|j |f|Ž| _)t||d|jd�| _*|dk�rt+|ƒnt ¡ | _,d S )Nré   r}   )r=   r>   rj   )rÑ   Ú1x1rf   )r&   r&   r&   rÑ   r&   r	  r   )Z	apply_act)r=   rj   )rÐ   )rÐ   r  rÿ   rj   rl   ZecarI   Zrd_channelsr’   r9   )-r•   r  r–   r   r   rK   rI   rO   r   r`   r5   r  rc   rÌ   r=   re   rj   ró   r   rÓ   rg   ru   rd   Úpre_normrh   rì   r   Ú	conv1_1x1rõ   rb   Ú	conv2_kxkrÜ   rm   r]   ro   rY   rq   rk   r   Úse_earlyrø   rl   Ú	conv3_1x1r   rè   )rR   r  r  rÐ   r  rç   rè   Únorm_act_layerZmid_chsrÿ   Ústride_poolZstride_1Zstride_2Z
dilation_2Úattn_kwargsrž   rS   rT   r–   Ò  sb    	    ÿ







     þ


zMbConvBlock.__init__ri   c                 C   s   t tt|d�| ƒ d S rû   ©r)   r   r  rý   rS   rS   rT   rþ     s    zMbConvBlock.init_weightsc                 C   sŠ   |   |¡}|  |¡}|  |¡}|  |¡}|  |¡}|  |¡}| jd k	rP|  |¡}|  |¡}| jd k	rn|  |¡}|  	|¡}|  
|¡| }|S rP   )ró   r
  rì   r  rõ   r  r  rø   rl   r  rè   ©rR   r´   ró   rS   rS   rT   r¼     s    











zMbConvBlock.forward)ri   )rV   rW   rX   rÁ   r.   rY   r
   r\   r–   rþ   r¼   r¾   rS   rS   rž   rT   r  Ï  s   ù
ù?
r  c                
       sZ   e Zd ZdZddddeƒ ddfeee eeeeef eee	dœ‡ fd	d
„Z
dd„ Z‡  ZS )ÚConvNeXtBlockz ConvNeXt Block
    Né   r&   r  Tr9   )r  r  rb   rÐ   r  rç   Úconv_mlprè   c	              	      s�  t ƒ  ¡  |p|}t|jƒ}	|r:tt|jƒ|jd�}
t}nd|jksHt	‚t
}
t}|| _|dkrlt||ƒ| _n*||krŒtj||d|jd�| _n
t ¡ | _|jdks¤t	‚d\}}|jdkr¼|}n|}|dkrÜt|||jd	�| _n
t ¡ | _t|||||d d
|jd�| _|
|ƒ| _||t|j| ƒ|j|	d�| _|�rR|j�rFt||jƒnt ¡ | _n|j�rft||jƒnt ¡ | _|dk�r‚t |ƒnt ¡ | _!d S )Nré   rL   r}   r&   )rb   r>   )rÑ   rf   r  rÑ   rí   T)rb   rÐ   r  Z	depthwiser>   )r>   rI   r9   )"r•   r–   r   rI   r   r   rK   rO   r   ru   r   r   Úuse_conv_mlprÌ   ró   r   r˜   re   rÓ   rg   rh   rì   r   Úconv_dwrë   rY   r5   rÛ   rG   rË   ÚlsrÂ   r   rè   )rR   r  r  rb   rÐ   r  rç   r  rè   rI   rK   Z	mlp_layerr  Z	stride_dwrž   rS   rT   r–   /  sL    




     þ
 zConvNeXtBlock.__init__c                 C   s”   |   |¡}|  |¡}|  |¡}| jrD|  |¡}|  |¡}|  |¡}n>| dddd¡}|  |¡}|  |¡}|  |¡}| dddd¡}|  |¡| }|S ©Nr   r}   ra   r&   )	ró   rì   r  r  rë   rÛ   r  Úpermuterè   r  rS   rS   rT   r¼   e  s    







zConvNeXtBlock.forward)rV   rW   rX   rÁ   r.   rY   r   r
   r[   r\   r–   r¼   r¾   rS   rS   rž   rT   r  +  s&   ÷
÷6r  ©rC   c                 C   s¨   | j \}}}}t||d  dkd|› d|d › d�ƒ t||d  dkdƒ |  |||d  |d ||d  |d |¡} |  ddddd	d
¡ ¡  d|d |d |¡}|S )Nr   úheight (ú) must be divisible by window (ú)r&   ri   ra   r}   é   r~   r¢   ©r¦   r    r§   r  r°   )r´   rC   rµ   r¶   r·   r€   ÚwindowsrS   rS   rT   Úwindow_partitionx  s    (,,r#  )rC   Úimg_sizec                 C   sf   |\}}| j d }|  d||d  ||d  |d |d |¡}| dddddd¡ ¡  d|||¡}|S )Nr¢   r   r&   ra   r}   r   r~   ©r¦   r§   r  r°   ©r"  rC   r$  r¶   r·   r€   r´   rS   rS   rT   Úwindow_reverse�  s
    
,$r'  )rD   c              	   C   s¦   | j \}}}}t||d  dkd|› d|d › �ƒ t||d  dkdƒ |  ||d ||d  |d ||d  |¡} |  dddddd	¡ ¡  d
|d |d |¡}|S )Nr   úheight ú must be divisible by grid r&   ri   r}   r   ra   r~   r¢   r!  )r´   rD   rµ   r¶   r·   r€   r"  rS   rS   rT   Úgrid_partitionŠ  s    &,,r*  )rD   r$  c                 C   sf   |\}}| j d }|  d||d  ||d  |d |d |¡}| dddddd¡ ¡  d|||¡}|S )Nr¢   r   r&   ra   r   r}   r~   r%  ©r"  rD   r$  r¶   r·   r€   r´   rS   rS   rT   Úgrid_reverse“  s
    
,$r,  )rç   c                 C   sR   d }| j dkr tt|| jd�}n.| j dkr8tt|d�}n| j dkrNtt|d�}|S )NrÛ   )rC   Z
hidden_dimr>   r  Úbias_tf)r?   r   r!   rA   r"   r#   )rç   rC   r�   rS   rS   rT   Úget_rel_pos_clsœ  s    


r.  c                       sF   e Zd ZdZdeƒ dfeeeedœ‡ fdd„Zdd„ Z	d	d
„ Z
‡  ZS )ÚPartitionAttentionClzR Grid or Block partition + Attn + FFN.
    NxC 'channels last' tensor layout.
    Úblockr9   ©rŽ   Úpartition_typerç   rè   c              
      s"  t ƒ  ¡  tt|jƒ|jd�}t|jƒ}|dk| _t	| jr@|j
n|jƒ| _t|| jƒ}||ƒ| _t|||j|j|j||j|jd�| _|jr˜t||jd�nt ¡ | _|dkr²t|ƒnt ¡ | _||ƒ| _t|t||j  ƒ||jd�| _!|jrøt||jd�nt ¡ | _"|dk�rt|ƒnt ¡ | _#d S ©Nré   r0  ©r2   r>   r3   r�   r:   r;   rî   r9   rï   )$r•   r–   r   r   rM   rO   r   rI   Úpartition_blockr   rC   rD   Úpartition_sizer.  rõ   r¿   r2   r8   r3   r:   r;   r»   rG   rÂ   r   rÓ   rö   r   r÷   rø   r   rY   r5   rÛ   rù   rú   ©rR   rŽ   r2  rç   rè   rK   rI   r�   rž   rS   rT   r–   ¬  s8    



ø

üzPartitionAttentionCl.__init__c                 C   s`   |j dd… }| jr"t|| jƒ}nt|| jƒ}|  |¡}| jrNt|| j|ƒ}nt|| j|ƒ}|S )Nr&   ra   )r¦   r5  r#  r6  r*  r»   r'  r,  ©rR   r´   r$  ZpartitionedrS   rS   rT   Ú_partition_attnÒ  s    
z$PartitionAttentionCl._partition_attnc              
   C   sD   ||   |  |  |  |¡¡¡¡ }||  |  |  |  |¡¡¡¡ }|S rP   ©r÷   rö   r9  rõ   rú   rù   rÛ   rø   rÔ   rS   rS   rT   r¼   á  s      zPartitionAttentionCl.forward©rV   rW   rX   rÁ   r/   rY   r]   r\   r–   r9  r¼   r¾   rS   rS   rž   rT   r/  §  s   ûû&r/  c                       sB   e Zd ZdZeƒ dfeeedœ‡ fdd„Zdd„ Zdd	„ Z	‡  Z
S )
ÚParallelPartitionAttentionzP Experimental. Grid and Block partition + single FFN
    NxC tensor layout.
    r9   ©rŽ   rç   rè   c              
      s\  t ƒ  ¡  |d dkst‚tt|jƒ|jd�}t|jƒ}|j	|j
ksHt‚t|j	ƒ| _t|| jƒ}||ƒ| _t||d |j|j|j||j|jd�| _t||d |j|j|j||j|jd�| _|jrÎt||jd�nt ¡ | _|dkrèt|ƒnt ¡ | _||ƒ| _t|t ||j! ƒ|||jd�| _"|j�r2t||jd�nt ¡ | _#|dk�rNt|ƒnt ¡ | _$d S )Nr}   r   ré   r4  rî   r9   )rð   rñ   Zout_featuresrI   rò   )%r•   r–   ru   r   r   rM   rO   r   rI   rC   rD   r   r6  r.  rõ   r¿   r2   r8   r3   r:   r;   Ú
attn_blockÚ	attn_gridrG   rÂ   r   rÓ   rö   r   r÷   rø   r   rY   r5   rÛ   rù   rú   )rR   rŽ   rç   rè   rK   rI   r�   rž   rS   rT   r–   ì  sP    


ø
ø

û z#ParallelPartitionAttention.__init__c                 C   sh   |j dd… }t|| jƒ}|  |¡}t|| j|ƒ}t|| jƒ}|  |¡}t|| j|ƒ}tj	||gdd�S )Nr&   ra   r¢   r£   )
r¦   r#  r6  r>  r'  r*  r?  r,  r¬   Úcat)rR   r´   r$  Zpartitioned_blockZx_windowZpartitioned_gridZx_gridrS   rS   rT   r9    s    

z*ParallelPartitionAttention._partition_attnc              
   C   sD   ||   |  |  |  |¡¡¡¡ }||  |  |  |  |¡¡¡¡ }|S rP   r:  rÔ   rS   rS   rT   r¼   *  s      z"ParallelPartitionAttention.forward)rV   rW   rX   rÁ   r/   rY   r\   r–   r9  r¼   r¾   rS   rS   rž   rT   r<  ç  s   üü1r<  c              	   C   s¨   | j \}}}}t||d  dkd|› d|d › d�ƒ t||d  dkdƒ |  ||||d  |d ||d  |d ¡} |  ddddd	d
¡ ¡  d||d |d ¡}|S )Nr   r  r  r  r&   ri   r}   r   ra   r~   r¢   r!  )r´   rC   rµ   r€   r¶   r·   r"  rS   rS   rT   Úwindow_partition_nchw0  s    (,,rA  c              	   C   sf   |\}}| j d }|  d||d  ||d  ||d |d ¡}| dddddd¡ ¡  d|||¡}|S )Nr&   r¢   r   ra   r   r}   r~   r%  r&  rS   rS   rT   Úwindow_reverse_nchw9  s
    
,$rB  c              
   C   s¦   | j \}}}}t||d  dkd|› d|d › �ƒ t||d  dkdƒ |  |||d ||d  |d ||d  ¡} |  dddddd	¡ ¡  d
||d |d ¡}|S )Nr   r(  r)  r&   ri   ra   r~   r}   r   r¢   r!  )r´   rD   rµ   r€   r¶   r·   r"  rS   rS   rT   Úgrid_partition_nchwB  s    &,,rC  c              	   C   sf   |\}}| j d }|  d||d  ||d  ||d |d ¡}| dddddd¡ ¡  d|||¡}|S )Nr&   r¢   r   ra   r   r~   r}   r%  r+  rS   rS   rT   Úgrid_reverse_nchwK  s
    
,$rD  c                       sF   e Zd ZdZdeƒ dfeeeedœ‡ fdd„Zdd„ Z	d	d
„ Z
‡  ZS )ÚPartitionAttention2dzH Grid or Block partition + Attn + FFN

    '2D' NCHW tensor layout.
    r0  r9   r1  c              
      s"  t ƒ  ¡  tt|jƒ|jd�}t|jƒ}|dk| _t	| jr@|j
n|jƒ| _t|| jƒ}||ƒ| _t|||j|j|j||j|jd�| _|jr˜t||jd�nt ¡ | _|dkr²t|ƒnt ¡ | _||ƒ| _t|t||j  ƒ||jd�| _!|jrøt||jd�nt ¡ | _"|dk�rt|ƒnt ¡ | _#d S r3  )$r•   r–   r   r   rK   rO   r   rI   r5  r   rC   rD   r6  r.  rõ   r‹   r2   r8   r3   r:   r;   r»   rG   rË   r   rÓ   rö   r   r÷   rø   r   rY   r5   rÛ   rù   rú   r7  rž   rS   rT   r–   Z  s8    



ø

üzPartitionAttention2d.__init__c                 C   s`   |j dd … }| jr"t|| jƒ}nt|| jƒ}|  |¡}| jrNt|| j|ƒ}nt|| j|ƒ}|S )Nr¤   )r¦   r5  rA  r6  rC  r»   rB  rD  r8  rS   rS   rT   r9  €  s    
z$PartitionAttention2d._partition_attnc              
   C   sD   ||   |  |  |  |¡¡¡¡ }||  |  |  |  |¡¡¡¡ }|S rP   r:  rÔ   rS   rS   rT   r¼   �  s      zPartitionAttention2d.forwardr;  rS   rS   rž   rT   rE  T  s   ûû&rE  c                       sP   e Zd ZdZdeƒ eƒ dfeeeeeedœ‡ fdd„Zddd	„Z	d
d„ Z
‡  ZS )ÚMaxxVitBlockz@ MaxVit conv, window partition + FFN , grid partition + FFN
    r&   r9   )rŽ   r�   rÐ   r†   r‡   rè   c           
         s‚   t ƒ  ¡  |j| _|jdkr tnt}||||||d�| _t|||d�}| jrPt	nt
}	|jr^d n|	f |Ž| _|	f ddi|—Ž| _d S )Nrr   ©rÐ   rç   rè   r=  r2  Úgrid)r•   r–   rF   Ú	nchw_attnr_   r  r  ÚconvÚdictrE  r/  rE   r>  r?  )
rR   rŽ   r�   rÐ   r†   r‡   rè   Úconv_clsr  Zpartition_layerrž   rS   rT   r–   ™  s    	
zMaxxVitBlock.__init__ri   c                 C   sJ   | j d k	rttt|d�| j ƒ ttt|d�| jƒ ttt|d�| jƒ d S rû   )r>  r)   r   rå   r?  r  rJ  rý   rS   rS   rT   rþ   ­  s    
zMaxxVitBlock.init_weightsc                 C   sX   |   |¡}| js | dddd¡}| jd k	r4|  |¡}|  |¡}| jsT| dddd¡}|S r  )rJ  rI  r  r>  r?  rÔ   rS   rS   rT   r¼   ³  s    



zMaxxVitBlock.forward)ri   )rV   rW   rX   rÁ   r.   r/   rY   r\   r–   rþ   r¼   r¾   rS   rS   rž   rT   rF  •  s   ùù
rF  c                       sJ   e Zd ZdZddeƒ eƒ dfeedœ‡ fdd„Zdd	d
„Zdd„ Z‡  Z	S )ÚParallelMaxxVitBlockzX MaxVit block with parallel cat(window + grid), one FF
    Experimental timm block.
    r&   r}   r9   ©r†   r‡   c           
         sŒ   t ƒ  ¡  |jdkrtnt}|dkrd||||||d�g}	|	|||||d�g|d  7 }	tj|	Ž | _n||||||d�| _t|||d�| _	d S )Nrr   r&   rG  )rç   rè   r=  )
r•   r–   r_   r  r  r   rô   rJ  r<  r»   )
rR   rŽ   r�   rÐ   Znum_convr†   r‡   rè   rL  Zconvsrž   rS   rT   r–   Æ  s    

zParallelMaxxVitBlock.__init__ri   c                 C   s,   t tt|d�| jƒ t tt|d�| jƒ d S rû   )r)   r   rå   r»   r  rJ  rý   rS   rS   rT   rþ   Û  s    z!ParallelMaxxVitBlock.init_weightsc                 C   s8   |   |¡}| dddd¡}|  |¡}| dddd¡}|S r  )rJ  r  r»   rÔ   rS   rS   rT   r¼   ß  s
    

zParallelMaxxVitBlock.forward)ri   )
rV   rW   rX   rÁ   r.   r/   r–   rþ   r¼   r¾   rS   rS   rž   rT   rM  Á  s   øù
rM  c                       sn   e Zd Zddddeƒ eƒ dfeeeeeeef eeee f eeee	e
e	 f dœ	‡ fdd„Zd	d
„ Z‡  ZS )ÚMaxxVitStager}   r   )é   rP  r€   r9   )	r  r  rÐ   ÚdepthÚ	feat_sizeÚblock_typesr‡   r†   rè   c
              
      s  t ƒ  ¡  d| _t||ƒ}g }
t|ƒD ]â\}}|dkr:|nd}|dksJt‚|dkr‚|jdkr`tnt}|
||||||	| d�g7 }
n‚|dkr´t	||ƒ}|
t
||||||	| d	�g7 }
nP|d
krÜ|
t||||||	| d�g7 }
n(|dk�r|
t||||||	| d�g7 }
|}q&tj|
Ž | _d S )NFr   r&   )r€   r�   ÚMÚPMr€   rr   rG  r�   )rÐ   r�   rç   rè   rT  )rÐ   r†   r‡   rè   rU  )r•   r–   Úgrad_checkpointingr   Ú	enumerateru   r_   r  r  r.  ræ   rF  rM  r   rô   Úblocks)rR   r  r  rÐ   rQ  rR  rS  r‡   r†   rè   rX  ÚiÚtZblock_striderL  r�   rž   rS   rT   r–   è  s\    

û
úú
ú
zMaxxVitStage.__init__c                 C   s,   | j rtj ¡ st| j|ƒ}n
|  |¡}|S rP   )rV  r¬   ÚjitZis_scriptingr*   rX  rÔ   rS   rS   rT   r¼   $  s    
zMaxxVitStage.forward)rV   rW   rX   r/   r.   rY   r
   r	   r]   r\   r   r–   r¼   r¾   rS   rS   rž   rT   rO  ç  s&   ö
ö<rO  c                
       sD   e Zd Zdeeeeeeeedœ‡ fdd	„Zdd
d„Zdd„ Z	‡  Z
S )ÚStemra   ri   FrH   rs   rt   )r  r  rb   rj   r>   rI   rK   rO   c	           
         sŒ   t ƒ  ¡  t|ttfƒs t|ƒ}tt||ƒ|d�}	|d | _d| _	t
||d |d||d�| _|	|d ƒ| _t
|d |d |d||d�| _d S )Nré   r¢   r}   r   )rÐ   rj   r>   r&   )r•   r–   rÜ   ÚlistÚtupler   r   r   r  rÐ   r   Úconv1rõ   Úconv2)
rR   r  r  rb   rj   r>   rI   rK   rO   r  rž   rS   rT   r–   .  s    

zStem.__init__c                 C   s   t tt|d�| ƒ d S rû   r  rý   rS   rS   rT   rþ   E  s    zStem.init_weightsc                 C   s"   |   |¡}|  |¡}|  |¡}|S rP   )r_  rõ   r`  rÔ   rS   rS   rT   r¼   H  s    


zStem.forward)ra   ri   FrH   rs   rt   )ri   )rV   rW   rX   rY   r]   r[   r\   r–   rþ   r¼   r¾   rS   rS   rž   rT   r\  ,  s$         ÷÷
r\  )rç   r$  c                 C   sF   | j d k	r| jst‚| S |d | j |d | j f}t| ||d�} | S )Nr   r&   )rC   rD   )rC   rD   ru   rB   r   )rç   r$  r6  rS   rS   rT   Úcfg_window_sizeO  s    

ra  c                 K   sŒ   i }i }i }|  ¡ D ]H\}}| d¡r8||| dd¡< q| d¡rT||| dd¡< q|||< qt| ft| jf|Žt| jf|Ždœ|—Ž} | S )NZtransformer_ri   Zconv_)r‡   r†   )ÚitemsÚ
startswithr   r‡   r†   )rç   ÚkwargsZtransformer_kwargsZconv_kwargsZbase_kwargsr¹   rº   rS   rS   rT   Ú_overlay_kwargsX  s$    


ÿýüre  c                	       s¼   e Zd ZdZd!eeeeeef f eeee	e	dœ‡ fdd	„Z
d"dd„Zejjdd„ ƒZejjd#dd„ƒZejjd$dd„ƒZejjdd„ ƒZd%dd„Zdd„ Zd&edœdd„Zdd „ Z‡  ZS )'r0   z| CoaTNet + MaxVit base model.

    Highly configurable for different block compositions, tensor layouts, pooling types.
    éà   ra   éè  rÏ   r9   )rç   r$  Úin_chansÚnum_classesÚglobal_poolÚ	drop_rateÚdrop_path_ratec                    sV  t ƒ  ¡  t|ƒ}|r"t|f|Ž}t|j|ƒ}	|| _|| _|jd  | _	| _|| _
d| _g | _t||j|jj|j|jj|jj|jjd�| _| jj}
|  jt| jjddd�g7  _tdd„ t|t|
ƒƒD ƒƒ}t|jƒ}t|jƒ|ksèt‚d	d„ t d
|t |jƒ¡ !|j¡D ƒ}| jj}g }t"|ƒD ]†}d‰ |j| }t‡ fdd„|D ƒƒ}|t#|||j| |j$| |j|	||| d�g7 }|
ˆ 9 }
|}|  jt||
d|› �d�g7  _�q"t%j&|Ž | _'t(t)|jjƒ|jjd�}|j*| _*| j*�rt% +¡ | _,t-| j	|| j*|||d�| _.n || j	ƒ| _,t/| j	|||d�| _.|j0dk�s4t‚|j0�rRt1t(| j2|j0d�| ƒ d S )Nr¢   F)r  r  rj   r>   rI   rK   rO   r}   Ústem)Znum_chsZ	reductionrâ   c                 S   s   g | ]\}}|| ‘qS rS   rS   )Ú.0rY  ÚsrS   rS   rT   Ú
<listcomp>”  s     z$MaxxVit.__init__.<locals>.<listcomp>c                 S   s   g | ]}|  ¡ ‘qS rS   )Útolist)rn  r´   rS   rS   rT   rp  ˜  s     r   c                    s   g | ]}|d  ˆ  d  ‘qS )r&   rS   )rn  Úr©Zstage_striderS   rT   rp  ž  s     )rQ  rS  r†   r‡   rR  rè   zstages.ré   )Zhidden_sizer=   rk  rK   )r=   rk  )ri   rÕ   rÙ   rÚ   r‰   rü   )3r•   r–   r   re  ra  r‡   ri  rj  r{   Znum_featuresrk  rV  Zfeature_infor\  rƒ   r†   rj   r„   rI   rK   rO   rm  rÐ   rK  r  r^  ÚzipÚlenr   ru   r¬   ZlinspaceÚsumÚsplitÚrangerO  r_   r   rô   Ústagesr   r   rˆ   rÓ   rë   r   Úheadr   rŠ   r)   Ú_init_weights)rR   rç   r$  rh  ri  rj  rk  rl  rd  r‡   rÐ   rR  Z
num_stagesZdprr  ry  rY  r  Zfinal_norm_layerrž   rs  rT   r–   r  s~    
ù	
&
ø

$
ú

zMaxxVit.__init__ri   c                 C   s<   t |dƒr8z|j|d� W n tk
r6   | ¡  Y nX d S )Nrþ   rü   )Úhasattrrþ   Ú	TypeError)rR   râ   rã   rä   rS   rS   rT   r{  Ä  s
    
zMaxxVit._init_weightsc                 C   s   dd„ |   ¡ D ƒS )Nc                    s*   h | ]"\‰ }t ‡ fd d„dD ƒƒrˆ ’qS )c                 3   s   | ]}|ˆ kV  qd S rP   rS   )rn  Ún©r¹   rS   rT   Ú	<genexpr>Ï  s     z4MaxxVit.no_weight_decay.<locals>.<setcomp>.<genexpr>)Úrelative_position_bias_tablezrel_pos.mlp)Úany)rn  Ú_rS   r  rT   Ú	<setcomp>Í  s   ÿz*MaxxVit.no_weight_decay.<locals>.<setcomp>)Znamed_parametersrQ   rS   rS   rT   Úno_weight_decayË  s    ÿzMaxxVit.no_weight_decayFc                 C   s   t dddgd�}|S )Nz^stem)z^stages\.(\d+)N)z^norm)iŸ† )rm  rX  )rK  )rR   ZcoarseZmatcherrS   rS   rT   Úgroup_matcherÑ  s
    þzMaxxVit.group_matcherTc                 C   s   | j D ]
}||_qd S rP   )ry  rV  )rR   Úenablero  rS   rS   rT   Úset_grad_checkpointingÙ  s    
zMaxxVit.set_grad_checkpointingc                 C   s   | j jS rP   )rz  ÚfcrQ   rS   rS   rT   Úget_classifierÞ  s    zMaxxVit.get_classifierNc                 C   s   || _ | j ||¡ d S rP   )ri  rz  Úreset)rR   ri  rj  rS   rS   rT   Úreset_classifierâ  s    zMaxxVit.reset_classifierc                 C   s"   |   |¡}|  |¡}|  |¡}|S rP   )rm  ry  rë   rÔ   rS   rS   rT   Úforward_featuresæ  s    


zMaxxVit.forward_features©Ú
pre_logitsc                 C   s   | j ||d�S )NrŽ  )rz  )rR   r´   r�  rS   rS   rT   Úforward_headì  s    zMaxxVit.forward_headc                 C   s   |   |¡}|  |¡}|S rP   )r�  r�  rÔ   rS   rS   rT   r¼   ï  s    

zMaxxVit.forward)rf  ra   rg  rÏ   r9   r9   )ri   )F)T)N)F)rV   rW   rX   rÁ   r-   r	   rY   r
   r]   r\   r–   r{  r¬   r[  Úignorer…  r†  rˆ  rŠ  rŒ  r�  r[   r�  r¼   r¾   rS   rS   rž   rT   r0   l  s:         øøR



rÑ   r<   FÚreluTrJ   rL   r>   r@   c                 C   s6   t t| |dd|||d|d�	td|||	|||
|d�d�S )NTFrn   )	rg   r=   rd   r`   re   rk   ro   rI   rK   )r6   r7   r=   rG   rK   rM   r?   rA   rN  ©rK  r.   r/   )rg   r=   Úconv_output_biasÚconv_attn_earlyÚconv_attn_act_layerÚconv_norm_layerÚtransformer_shortcut_biasÚtransformer_norm_layerÚtransformer_norm_layer_clrG   r?   rA   rS   rS   rT   Ú_rw_coat_cfgõ  s.    ÷øôr›  rf   ç      °?r1   c                 C   s4   t t| |d||d|d�td||||	|||
|d�	d�S )NFrn   )rg   r=   r`   re   rq   rI   rK   )	r6   r=   r2   rC   rG   rK   rM   r?   rA   rN  r“  )rg   r=   r”  Úconv_attn_ratior—  r™  rš  rC   r2   rG   r?   rA   rS   rS   rT   Ú_rw_max_cfg)  s,    ù	÷örž  rN   rÛ   c                 C   sD   t |ƒ}ttd| |d|d ||d�td||||d |||	|
d�	d�S )Nrr   Fr   )r_   rg   r=   r`   rG   rK   rM   r&   )	r6   r=   rC   rE   rG   rK   rM   r?   rA   rN  )r   rK  r.   r/   )rg   r=   r—  Zconv_norm_layer_clr™  rš  rC   rE   rG   r?   rA   rS   rS   rT   Ú	_next_cfgU  s.    ù	÷örŸ  c                   C   s"   t tdddd�tddddd�d	�S )
Ngü©ñÒMbP?Z	gelu_tanhZsame)rO   rI   rj   rt   Fr-  )rO   rI   r3   r?   rN  r“  rS   rS   rS   rT   Ú_tf_cfg|  s    ýüúr   )r‚   é€   é   r@   r|   )r1   r‚   )r{   r   rƒ   rp   )r”  r�  )ra   r   é   ra   )rg   r”  r�  rv   )r}   ra   r  r}   )r•  r˜  )r}   r£  rP  r}   )rg   r•  r˜  )r¡  r¢  r@   é   )r‚   r¡  rn   )rg   r–  )rx   ry   rz   é   )rw   rx   )rg   r–  rG   rs   )rg   r•  r˜  r™  ry   )r”  r�  r?   rA   )rg   r?   rÍ   )r=   r•  r˜  r?   rA   )rg   r?   rA   )rg   r–  rG   r?   )r€   r€   ©r€   r�   r¦  )r{   r   rƒ   r_   rÕ   )r{   r   rƒ   rŠ   )rt   N)r?   rG   r‚   rz   )r{   r   rƒ   rˆ   r¡  r¤  rx   r¥  )r}   é   é   r}   )r¢  r@   é   é   rª  )r1   r‚   r¡  r¢  )r}   r}   r~   r}   )rT  rT  rT  rT  )é   r1   )r{   r   r_   rƒ   )r&   r}   ra   r&   )rU  rU  rU  rU  )r?   )r{   r   r_   rƒ   rˆ   )r{   r   r_   rƒ   rŠ   )é0   rw   )rE   r?   )r}   r£  r§  r}   )rE   )é    i@  i€  r©  )r}   r£  é   r}   )éP   r­  r©  )r{   r   r_   rƒ   r„   rˆ   ))Zcoatnet_pico_rwZcoatnet_nano_rwZcoatnet_0_rwZcoatnet_1_rwZcoatnet_2_rwZcoatnet_3_rwZcoatnet_bn_0_rwZcoatnet_rmlp_nano_rwZcoatnet_rmlp_0_rwZcoatnet_rmlp_1_rwZcoatnet_rmlp_1_rw2Zcoatnet_rmlp_2_rwZcoatnet_rmlp_3_rwZcoatnet_nano_ccZcoatnext_nano_rwZ	coatnet_0Z	coatnet_1Z	coatnet_2Z	coatnet_3Z	coatnet_4Z	coatnet_5Zmaxvit_pico_rwZmaxvit_nano_rwZmaxvit_tiny_rwZmaxvit_tiny_pmZmaxvit_rmlp_pico_rwZmaxvit_rmlp_nano_rwZmaxvit_rmlp_tiny_rwZmaxvit_rmlp_small_rwZmaxvit_rmlp_base_rwZmaxxvit_rmlp_nano_rwZmaxxvit_rmlp_tiny_rwZmaxxvit_rmlp_small_rwZmaxxvitv2_nano_rwZmaxxvitv2_rmlp_base_rwZmaxxvitv2_rmlp_large_rwÚmaxvit_tiny_tfÚmaxvit_small_tfÚmaxvit_base_tfÚmaxvit_large_tfÚmaxvit_xlarge_tf)Úmodelc                 C   sÆ   |  ¡ }i }|  ¡ D ]¬\}}| d¡rn| |d d… ¡}|j|jjksZ|jd |jd krnt||j|jjd�}||kr¸|j|| jkr¸| 	¡ ||  	¡ kr¸|jdks¨t
‚| || j¡}|||< q|S )Nr�  iãÿÿÿr   r&   )Znew_window_sizeZnew_bias_shape)r}   r   )Ú
state_dictrb  ÚendswithZget_submoduler¦   r�  rC   r%   ÚndimZnumelru   r©   )r¶  rµ  Zmodel_state_dictZout_dictr¹   rº   ÚmrS   rS   rT   Úcheckpoint_filter_fný  s     
"ý,
rº  c                 K   sT   |d kr.| t kr| }nd |  d¡d d… ¡}tt| |ft | tdd�tdœ|—ŽS )Nrƒ  r¢   T)Zflatten_sequential)Z	model_cfgZfeature_cfgZpretrained_filter_fn)Ú
model_cfgsÚjoinrw  r'   r0   rK  rº  )ÚvariantZcfg_variantÚ
pretrainedrd  rS   rS   rT   Ú_create_maxxvit  s      ÿüûr¿  c                 K   s    | dddddddddd	d
œ|–S )Nrg  )ra   rf  rf  )r  r  çffffffî?Zbicubic)ç      à?rÁ  rÁ  z
stem.conv1zhead.fcT)Úurlri  Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚmeanrØ   Z
first_convÚ
classifierZfixed_input_sizerS   )rÂ  rd  rS   rS   rT   Ú_cfg   s          ûúrÉ  )rÂ  ztimm/zyhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_nano_rw_224_sw-f53093b4.pthgÍÌÌÌÌÌì?)Ú	hf_hub_idrÂ  rÅ  zvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_0_rw_224_sw-a6439706.pth)rÊ  rÂ  zvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_1_rw_224_sw-5cae1ea8.pth)rÊ  )ra   ry   ry   )r§  r§  g      ð?Zsquash)rÊ  rÃ  rÄ  rÅ  Ú	crop_modezyhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_bn_0_rw_224_sw-c228e218.pthrÀ  )rÊ  rÂ  rÇ  rØ   rÅ  z~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_nano_rw_224_sw-bd1d51b3.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_1_rw_224_sw-9051e6c3.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_2_rw_224_sw-5ccfac55.pthzzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnext_nano_rw_224_ad-22cb71c2.pthi-.  )rÊ  ri  )ra   r¢  r¢  )é   rÌ  )rÂ  rÃ  rÄ  zxhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_nano_rw_256_sw-fb127241.pth)rÊ  rÂ  rÃ  rÄ  zxhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_tiny_rw_224_sw-7d0dffeb.pthz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_pico_rw_256_sw-8d82f2c6.pthz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_nano_rw_256_sw-c17bb0d6.pthz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_tiny_rw_256_sw-bbef0ff5.pthz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_small_rw_224_sw-6ef0ae4f.pthz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxxvit_rmlp_nano_rw_256_sw-0325d459.pthzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxxvit_rmlp_small_rw_256_sw-37e217ff.pth)rÊ  rÃ  rÄ  )rÊ  rÇ  rØ   )ra   r@   r@   )r®  r®  iSU  )rÊ  rÃ  rÅ  rË  )Dzcoatnet_pico_rw_224.untrainedzcoatnet_nano_rw_224.sw_in1kzcoatnet_0_rw_224.sw_in1kzcoatnet_1_rw_224.sw_in1kz!coatnet_2_rw_224.sw_in12k_ft_in1kz'coatnet_rmlp_1_rw2_224.sw_in12k_ft_in1kz&coatnet_rmlp_2_rw_224.sw_in12k_ft_in1kz&coatnet_rmlp_2_rw_384.sw_in12k_ft_in1kzcoatnet_bn_0_rw_224.sw_in1kz coatnet_rmlp_nano_rw_224.sw_in1kzcoatnet_rmlp_0_rw_224.untrainedzcoatnet_rmlp_1_rw_224.sw_in1kzcoatnet_rmlp_2_rw_224.sw_in1kzcoatnet_rmlp_3_rw_224.untrainedzcoatnet_nano_cc_224.untrainedzcoatnext_nano_rw_224.sw_in1kzcoatnet_2_rw_224.sw_in12kzcoatnet_3_rw_224.sw_in12kzcoatnet_rmlp_1_rw2_224.sw_in12kzcoatnet_rmlp_2_rw_224.sw_in12kzcoatnet_0_224.untrainedzcoatnet_1_224.untrainedzcoatnet_2_224.untrainedzcoatnet_3_224.untrainedzcoatnet_4_224.untrainedzcoatnet_5_224.untrainedzmaxvit_pico_rw_256.untrainedzmaxvit_nano_rw_256.sw_in1kzmaxvit_tiny_rw_224.sw_in1kzmaxvit_tiny_rw_256.untrainedzmaxvit_tiny_pm_256.untrainedzmaxvit_rmlp_pico_rw_256.sw_in1kzmaxvit_rmlp_nano_rw_256.sw_in1kzmaxvit_rmlp_tiny_rw_256.sw_in1kz maxvit_rmlp_small_rw_224.sw_in1kz"maxvit_rmlp_small_rw_256.untrainedz(maxvit_rmlp_base_rw_224.sw_in12k_ft_in1kz(maxvit_rmlp_base_rw_384.sw_in12k_ft_in1kz maxvit_rmlp_base_rw_224.sw_in12kz maxxvit_rmlp_nano_rw_256.sw_in1kz"maxxvit_rmlp_tiny_rw_256.untrainedz!maxxvit_rmlp_small_rw_256.sw_in1kzmaxxvitv2_nano_rw_256.sw_in1kz+maxxvitv2_rmlp_base_rw_224.sw_in12k_ft_in1kz+maxxvitv2_rmlp_base_rw_384.sw_in12k_ft_in1kz%maxxvitv2_rmlp_large_rw_224.untrainedz#maxxvitv2_rmlp_base_rw_224.sw_in12kzmaxvit_tiny_tf_224.in1kzmaxvit_tiny_tf_384.in1kzmaxvit_tiny_tf_512.in1kzmaxvit_small_tf_224.in1kzmaxvit_small_tf_384.in1kzmaxvit_small_tf_512.in1kzmaxvit_base_tf_224.in1kzmaxvit_base_tf_384.in1kzmaxvit_base_tf_512.in1kzmaxvit_large_tf_224.in1kzmaxvit_large_tf_384.in1kzmaxvit_large_tf_512.in1kzmaxvit_base_tf_224.in21kz maxvit_base_tf_384.in21k_ft_in1kz maxvit_base_tf_512.in21k_ft_in1kzmaxvit_large_tf_224.in21kz!maxvit_large_tf_384.in21k_ft_in1kz!maxvit_large_tf_512.in21k_ft_in1kzmaxvit_xlarge_tf_224.in21kz"maxvit_xlarge_tf_384.in21k_ft_in1kz"maxvit_xlarge_tf_512.in21k_ft_in1k)Úreturnc                 K   s   t dd| i|—ŽS )NÚcoatnet_pico_rw_224r¾  )rÎ  ©r¿  ©r¾  rd  rS   rS   rT   rÎ    s    rÎ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_nano_rw_224r¾  )rÑ  rÏ  rÐ  rS   rS   rT   rÑ    s    rÑ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_0_rw_224r¾  )rÒ  rÏ  rÐ  rS   rS   rT   rÒ    s    rÒ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_1_rw_224r¾  )rÓ  rÏ  rÐ  rS   rS   rT   rÓ    s    rÓ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_2_rw_224r¾  )rÔ  rÏ  rÐ  rS   rS   rT   rÔ    s    rÔ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_3_rw_224r¾  )rÕ  rÏ  rÐ  rS   rS   rT   rÕ    s    rÕ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_bn_0_rw_224r¾  )rÖ  rÏ  rÐ  rS   rS   rT   rÖ  $  s    rÖ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_rmlp_nano_rw_224r¾  )r×  rÏ  rÐ  rS   rS   rT   r×  )  s    r×  c                 K   s   t dd| i|—ŽS )NÚcoatnet_rmlp_0_rw_224r¾  )rØ  rÏ  rÐ  rS   rS   rT   rØ  .  s    rØ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_rmlp_1_rw_224r¾  )rÙ  rÏ  rÐ  rS   rS   rT   rÙ  3  s    rÙ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_rmlp_1_rw2_224r¾  )rÚ  rÏ  rÐ  rS   rS   rT   rÚ  8  s    rÚ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_rmlp_2_rw_224r¾  )rÛ  rÏ  rÐ  rS   rS   rT   rÛ  =  s    rÛ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_rmlp_2_rw_384r¾  )rÜ  rÏ  rÐ  rS   rS   rT   rÜ  B  s    rÜ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_rmlp_3_rw_224r¾  )rÝ  rÏ  rÐ  rS   rS   rT   rÝ  G  s    rÝ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_nano_cc_224r¾  )rÞ  rÏ  rÐ  rS   rS   rT   rÞ  L  s    rÞ  c                 K   s   t dd| i|—ŽS )NÚcoatnext_nano_rw_224r¾  )rß  rÏ  rÐ  rS   rS   rT   rß  Q  s    rß  c                 K   s   t dd| i|—ŽS )NÚcoatnet_0_224r¾  )rà  rÏ  rÐ  rS   rS   rT   rà  V  s    rà  c                 K   s   t dd| i|—ŽS )NÚcoatnet_1_224r¾  )rá  rÏ  rÐ  rS   rS   rT   rá  [  s    rá  c                 K   s   t dd| i|—ŽS )NÚcoatnet_2_224r¾  )râ  rÏ  rÐ  rS   rS   rT   râ  `  s    râ  c                 K   s   t dd| i|—ŽS )NÚcoatnet_3_224r¾  )rã  rÏ  rÐ  rS   rS   rT   rã  e  s    rã  c                 K   s   t dd| i|—ŽS )NÚcoatnet_4_224r¾  )rä  rÏ  rÐ  rS   rS   rT   rä  j  s    rä  c                 K   s   t dd| i|—ŽS )NÚcoatnet_5_224r¾  )rå  rÏ  rÐ  rS   rS   rT   rå  o  s    rå  c                 K   s   t dd| i|—ŽS )NÚmaxvit_pico_rw_256r¾  )ræ  rÏ  rÐ  rS   rS   rT   ræ  t  s    ræ  c                 K   s   t dd| i|—ŽS )NÚmaxvit_nano_rw_256r¾  )rç  rÏ  rÐ  rS   rS   rT   rç  y  s    rç  c                 K   s   t dd| i|—ŽS )NÚmaxvit_tiny_rw_224r¾  )rè  rÏ  rÐ  rS   rS   rT   rè  ~  s    rè  c                 K   s   t dd| i|—ŽS )NÚmaxvit_tiny_rw_256r¾  )ré  rÏ  rÐ  rS   rS   rT   ré  ƒ  s    ré  c                 K   s   t dd| i|—ŽS )NÚmaxvit_rmlp_pico_rw_256r¾  )rê  rÏ  rÐ  rS   rS   rT   rê  ˆ  s    rê  c                 K   s   t dd| i|—ŽS )NÚmaxvit_rmlp_nano_rw_256r¾  )rë  rÏ  rÐ  rS   rS   rT   rë  �  s    rë  c                 K   s   t dd| i|—ŽS )NÚmaxvit_rmlp_tiny_rw_256r¾  )rì  rÏ  rÐ  rS   rS   rT   rì  ’  s    rì  c                 K   s   t dd| i|—ŽS )NÚmaxvit_rmlp_small_rw_224r¾  )rí  rÏ  rÐ  rS   rS   rT   rí  —  s    rí  c                 K   s   t dd| i|—ŽS )NÚmaxvit_rmlp_small_rw_256r¾  )rî  rÏ  rÐ  rS   rS   rT   rî  œ  s    rî  c                 K   s   t dd| i|—ŽS )NÚmaxvit_rmlp_base_rw_224r¾  )rï  rÏ  rÐ  rS   rS   rT   rï  ¡  s    rï  c                 K   s   t dd| i|—ŽS )NÚmaxvit_rmlp_base_rw_384r¾  )rð  rÏ  rÐ  rS   rS   rT   rð  ¦  s    rð  c                 K   s   t dd| i|—ŽS )NÚmaxvit_tiny_pm_256r¾  )rñ  rÏ  rÐ  rS   rS   rT   rñ  «  s    rñ  c                 K   s   t dd| i|—ŽS )NÚmaxxvit_rmlp_nano_rw_256r¾  )rò  rÏ  rÐ  rS   rS   rT   rò  °  s    rò  c                 K   s   t dd| i|—ŽS )NÚmaxxvit_rmlp_tiny_rw_256r¾  )ró  rÏ  rÐ  rS   rS   rT   ró  µ  s    ró  c                 K   s   t dd| i|—ŽS )NÚmaxxvit_rmlp_small_rw_256r¾  )rô  rÏ  rÐ  rS   rS   rT   rô  º  s    rô  c                 K   s   t dd| i|—ŽS )NÚmaxxvitv2_nano_rw_256r¾  )rõ  rÏ  rÐ  rS   rS   rT   rõ  ¿  s    rõ  c                 K   s   t dd| i|—ŽS )NÚmaxxvitv2_rmlp_base_rw_224r¾  )rö  rÏ  rÐ  rS   rS   rT   rö  Ä  s    rö  c                 K   s   t dd| i|—ŽS )NÚmaxxvitv2_rmlp_base_rw_384r¾  )r÷  rÏ  rÐ  rS   rS   rT   r÷  É  s    r÷  c                 K   s   t dd| i|—ŽS )NÚmaxxvitv2_rmlp_large_rw_224r¾  )rø  rÏ  rÐ  rS   rS   rT   rø  Î  s    rø  c                 K   s   t dd| i|—ŽS )NÚmaxvit_tiny_tf_224r°  r¾  )rù  r°  rÏ  rÐ  rS   rS   rT   rù  Ó  s    rù  c                 K   s   t dd| i|—ŽS )NÚmaxvit_tiny_tf_384r°  r¾  )rú  r°  rÏ  rÐ  rS   rS   rT   rú  Ø  s    rú  c                 K   s   t dd| i|—ŽS )NÚmaxvit_tiny_tf_512r°  r¾  )rû  r°  rÏ  rÐ  rS   rS   rT   rû  Ý  s    rû  c                 K   s   t dd| i|—ŽS )NÚmaxvit_small_tf_224r±  r¾  )rü  r±  rÏ  rÐ  rS   rS   rT   rü  â  s    rü  c                 K   s   t dd| i|—ŽS )NÚmaxvit_small_tf_384r±  r¾  )rý  r±  rÏ  rÐ  rS   rS   rT   rý  ç  s    rý  c                 K   s   t dd| i|—ŽS )NÚmaxvit_small_tf_512r±  r¾  )rþ  r±  rÏ  rÐ  rS   rS   rT   rþ  ì  s    rþ  c                 K   s   t dd| i|—ŽS )NÚmaxvit_base_tf_224r²  r¾  )rÿ  r²  rÏ  rÐ  rS   rS   rT   rÿ  ñ  s    rÿ  c                 K   s   t dd| i|—ŽS )NÚmaxvit_base_tf_384r²  r¾  )r   r²  rÏ  rÐ  rS   rS   rT   r   ö  s    r   c                 K   s   t dd| i|—ŽS )NÚmaxvit_base_tf_512r²  r¾  )r  r²  rÏ  rÐ  rS   rS   rT   r  û  s    r  c                 K   s   t dd| i|—ŽS )NÚmaxvit_large_tf_224r³  r¾  )r  r³  rÏ  rÐ  rS   rS   rT   r   	  s    r  c                 K   s   t dd| i|—ŽS )NÚmaxvit_large_tf_384r³  r¾  )r  r³  rÏ  rÐ  rS   rS   rT   r  	  s    r  c                 K   s   t dd| i|—ŽS )NÚmaxvit_large_tf_512r³  r¾  )r  r³  rÏ  rÐ  rS   rS   rT   r  
	  s    r  c                 K   s   t dd| i|—ŽS )NÚmaxvit_xlarge_tf_224r´  r¾  )r  r´  rÏ  rÐ  rS   rS   rT   r  	  s    r  c                 K   s   t dd| i|—ŽS )NÚmaxvit_xlarge_tf_384r´  r¾  )r  r´  rÏ  rÐ  rS   rS   rT   r  	  s    r  c                 K   s   t dd| i|—ŽS )NÚmaxvit_xlarge_tf_512r´  r¾  )r  r´  rÏ  rÐ  rS   rS   rT   r  	  s    r  )ri   )ri   )rÑ   r<   FFr’  ri   TrJ   rL   Nr>   r@   )rf   r<   Frœ  ri   rJ   rL   Nr1   Nr>   r@   )rf   r<   rJ   rL   rJ   rL   NFrN   rÛ   r@   )NF)ri   )F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)žrÁ   r   Úcollectionsr   Údataclassesr   r   r   Ú	functoolsr   Útypingr   r   r	   r
   r   r¬   r   Z	torch.jitr   Z	timm.datar   r   Ztimm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   Z_builderr'   Z_features_fxr(   Z_manipulater)   r*   Ú	_registryr+   r,   Ú__all__r/   r.   r-   ÚModuler‹   r¿   rÂ   rË   rÌ   rå   ræ   r  r  r  r  rY   r#  r'  r*  r,  r.  r/  r<  rA  rB  rC  rD  rE  rF  rM  rO  r\  ra  re  r0   r›  rž  rŸ  r   rK  r»  rº  r¿  rÉ  Zdefault_cfgsrÎ  rÑ  rÒ  rÓ  rÔ  rÕ  rÖ  r×  rØ  rÙ  rÚ  rÛ  rÜ  rÝ  rÞ  rß  rà  rá  râ  rã  rä  rå  ræ  rç  rè  ré  rê  rë  rì  rí  rî  rï  rð  rñ  rò  ró  rô  rõ  rö  r÷  rø  rù  rú  rû  rü  rý  rþ  rÿ  r   r  r  r  r  r  r  r  rS   rS   rS   rT   Ú<module>   s  $   #EF'A	\M		@I		A,&E#	            ô
5           ô
-          õ
'ýþü	ýýü
ýþü	ýýü
ýþü
ýýüýüüýüüýþü	ýûüýýü
ýüüýüüüûüþûüüüüüüüûüûüûüûüûüûüûüþû
ûÿúûúüûüûûþúüÿû	ûÿúúù	úù	úù	úù	úù€ € ›  sýþþÿÿÿ   þ üýþþýþþþþ ýþ þ ý ý ýý þÿ   þþ ý ý þÿ   þþ þ   þ   þ þ   þ   þ þ   þ   þ þ   þ   þþ   þ   þþ   þ  þþ   þ   þ€ « \