U
    ¯Ê¹j´~  ã                   @   sš  d Z ddlmZ ddlmZ ddlmZ ddlZddlm	Z	 ddl
mZmZ ddlmZmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZmZ dgZG dd„ de	jƒZG dd„ de	jƒZG dd„ de	jƒZG dd„ de	j ƒZ!G dd„ de	j ƒZ"G dd„ de	jƒZ#G dd„ de	jƒZ$G dd„ de	jƒZ%G dd„ de	jƒZ&G dd „ d e	jƒZ'G d!d"„ d"e	jƒZ(G d#d$„ d$e	jƒZ)G d%d„ de	jƒZ*G d&d'„ d'e*ƒZ+d(d)„ Z,e-e-d*d+d,d-d.�e-d*d+d/d0d.�e-d1d2d3d0d.�e-d4d2d,d0d.�e-d5d2d6d0d.�e-d5d2d6d0d7d8d9�e-d:d;d<d0d7d8d9�e-d=d;d>d0d7d?�e-d4d2d,d@d7d?�e-dAd;dBd@d7d?�dC�
Z.d{dFdG„Z/d|dIdJ„Z0ee0dKdL�e0dKdL�e0dKdL�e0dKdL�e0dKdL�e0dKdMdN�e0dKdMdN�e0dKdMdN�e0dKdMdN�e0dKdMdN�e0dOdP�e0dOdP�e0dOdP�e0dOdP�e0dOdP�e0dOdP�e0dOdP�e0dOdP�e0dOdP�e0dOdP�dQœƒZ1ed}e*dRœdSdT„ƒZ2ed~e*dRœdUdV„ƒZ3ede*dRœdWdX„ƒZ4ed€e*dRœdYdZ„ƒZ5ed�e*dRœd[d\„ƒZ6ed‚e*dRœd]d^„ƒZ7edƒe*dRœd_d`„ƒZ8ed„e*dRœdadb„ƒZ9ed…e*dRœdcdd„ƒZ:ed†e*dRœdedf„ƒZ;ed‡e*dRœdgdh„ƒZ<edˆe*dRœdidj„ƒZ=ed‰e*dRœdkdl„ƒZ>edŠe*dRœdmdn„ƒZ?ed‹e*dRœdodp„ƒZ@edŒe*dRœdqdr„ƒZAed�e*dRœdsdt„ƒZBedŽe*dRœdudv„ƒZCed�e*dRœdwdx„ƒZDed�e*dRœdydz„ƒZEdS )‘aÈ   LeViT

Paper: `LeViT: a Vision Transformer in ConvNet's Clothing for Faster Inference`
    - https://arxiv.org/abs/2104.01136

@article{graham2021levit,
  title={LeViT: a Vision Transformer in ConvNet's Clothing for Faster Inference},
  author={Benjamin Graham and Alaaeldin El-Nouby and Hugo Touvron and Pierre Stock and Armand Joulin and Herv'e J'egou and Matthijs Douze},
  journal={arXiv preprint arXiv:22104.01136},
  year={2021}
}

Adapted from official impl at https://github.com/facebookresearch/LeViT, original copyright bellow.

This version combines both conv/linear models and fixes torchscript compatibility.

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
é    )ÚOrderedDict)Úpartial)ÚDictN)ÚIMAGENET_DEFAULT_STDÚIMAGENET_DEFAULT_MEAN)Ú	to_ntupleÚ	to_2tupleÚget_act_layerÚDropPathÚtrunc_normal_Úndgridé   )Úbuild_model_with_cfg)Úcheckpoint_seq)Úgenerate_default_cfgsÚregister_modelÚLevitc                       s6   e Zd Zd	‡ fdd„	Ze ¡ dd„ ƒZdd„ Z‡  ZS )
ÚConvNormr   r   c	           	   
      sH   t ƒ  ¡  tj|||||||dd�| _t |¡| _tj | jj	|¡ d S ©NF©Úbias)
ÚsuperÚ__init__ÚnnÚConv2dÚlinearZBatchNorm2dÚbnÚinitÚ	constant_Úweight)	ÚselfÚin_chsÚout_chsÚkernel_sizeÚstrideÚpaddingÚdilationÚgroupsÚbn_weight_init©Ú	__class__© úP/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/levit.pyr   +   s    
zConvNorm.__init__c              	   C   s¾   | j | j }}|j|j|j d  }|j|d d …d d d f  }|j|j|j |j|j d   }tj| 	d¡| 	d¡|j
dd … | j j| j j| j j| j jd�}|jj |¡ |jj |¡ |S )Nç      à?r   r   é   )r$   r%   r&   r'   )r   r   r   Úrunning_varÚepsr   Úrunning_meanr   r   ÚsizeÚshaper$   r%   r&   r'   ÚdataÚcopy_)r    Úcr   ÚwÚbÚmr+   r+   r,   Úfuse3   s     "     þzConvNorm.fusec                 C   s   |   |  |¡¡S ©N)r   r   ©r    Úxr+   r+   r,   Úforward@   s    zConvNorm.forward)r   r   r   r   r   r   ©	Ú__name__Ú
__module__Ú__qualname__r   ÚtorchÚno_gradr:   r>   Ú__classcell__r+   r+   r)   r,   r   *   s              ÿ
r   c                       s6   e Zd Zd‡ fdd„	Ze ¡ dd„ ƒZdd„ Z‡  ZS )	Ú
LinearNormr   c                    s>   t ƒ  ¡  tj||dd�| _t |¡| _tj | jj	|¡ d S r   )
r   r   r   ÚLinearr   ÚBatchNorm1dr   r   r   r   )r    Úin_featuresÚout_featuresr(   r)   r+   r,   r   E   s    
zLinearNorm.__init__c                 C   s”   | j | j }}|j|j|j d  }|j|d d …d f  }|j|j|j |j|j d   }t | 	d¡| 	d¡¡}|jj
 |¡ |jj
 |¡ |S )Nr-   r   r   )r   r   r   r/   r0   r   r1   r   rG   r2   r4   r5   )r    Úlr   r7   r8   r9   r+   r+   r,   r:   L   s    "zLinearNorm.fusec                 C   s"   |   |¡}|  | dd¡¡ |¡S )Nr   r   )r   r   ÚflattenZ
reshape_asr<   r+   r+   r,   r>   W   s    
zLinearNorm.forward)r   r?   r+   r+   r)   r,   rF   D   s   

rF   c                       s6   e Zd Zd
‡ fdd„	Ze ¡ dd„ ƒZdd	„ Z‡  ZS )Ú
NormLinearTç{®Gáz”?ç        c                    sf   t ƒ  ¡  t |¡| _t |¡| _tj|||d�| _t	| jj
|d� | jjd k	rbtj | jjd¡ d S )Nr   )Ústdr   )r   r   r   rH   r   ÚDropoutÚdroprG   r   r   r   r   r   r   )r    rI   rJ   r   rP   rR   r)   r+   r,   r   ]   s    
zNormLinear.__init__c                 C   sÖ   | j | j }}|j|j|j d  }|j| j j| j j |j|j d   }|j|d d d …f  }|jd krz|| jjj }n$|j|d d …d f   d¡| jj }t	 
| d¡| d¡¡}|jj |¡ |jj |¡ |S )Nr-   éÿÿÿÿr   r   )r   r   r   r/   r0   r   r1   ÚTÚviewr   rG   r2   r4   r5   )r    r   rK   r7   r8   r9   r+   r+   r,   r:   g   s    &
$zNormLinear.fusec                 C   s   |   |  |  |¡¡¡S r;   )r   rR   r   r<   r+   r+   r,   r>   v   s    zNormLinear.forward)TrN   rO   r?   r+   r+   r)   r,   rM   \   s   

rM   c                       s   e Zd Z‡ fdd„Z‡  ZS )ÚStem8c              
      sŽ   t ƒ  ¡  d| _|  dt||d dddd�¡ |  d|ƒ ¡ |  d	t|d |d dddd�¡ |  d
|ƒ ¡ |  dt|d |dddd�¡ d S )Né   Úconv1é   é   r.   r   ©r$   r%   Úact1Úconv2Úact2Úconv3©r   r   r$   Z
add_moduler   ©r    r!   r"   Ú	act_layerr)   r+   r,   r   {   s    
"zStem8.__init__©r@   rA   rB   r   rE   r+   r+   r)   r,   rV   z   s   rV   c                       s   e Zd Z‡ fdd„Z‡  ZS )ÚStem16c              
      s¾   t ƒ  ¡  d| _|  dt||d dddd�¡ |  d|ƒ ¡ |  d	t|d |d
 dddd�¡ |  d|ƒ ¡ |  dt|d
 |d dddd�¡ |  d|ƒ ¡ |  dt|d |dddd�¡ d S )Né   rX   rW   rZ   r.   r   r[   r\   r]   rY   r^   r_   Zact3Zconv4r`   ra   r)   r+   r,   r   ‡   s    
""zStem16.__init__rc   r+   r+   r)   r,   rd   †   s   rd   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )Ú
DownsampleFc                    s:   t ƒ  ¡  || _t|ƒ| _|r0tjd|ddd�nd | _d S )NrZ   r   F)r$   r%   Úcount_include_pad)r   r   r$   r   Ú
resolutionr   Ú	AvgPool2dÚpool)r    r$   rh   Úuse_poolr)   r+   r,   r   •   s    

zDownsample.__init__c                 C   s†   |j \}}}| || jd | jd |¡}| jd k	rV|  | dddd¡¡ dddd¡}n"|d d …d d | j…d d | j…f }| |d|¡S )Nr   r   rZ   r.   rS   )r3   rU   rh   rj   Úpermuter$   Úreshape)r    r=   ÚBÚNÚCr+   r+   r,   r>   ›   s    
$"zDownsample.forward)F)r@   rA   rB   r   r>   rE   r+   r+   r)   r,   rf   ”   s   rf   c                       sp   e Zd ZU eeejf ed< ddddej	f‡ fdd„	Z
e ¡ d‡ fd	d
„	ƒZejejdœdd„Zdd„ Z‡  ZS )Ú	AttentionÚattention_bias_cacherW   ç      @é   Fc              
      sJ  t ƒ  ¡  |rtnt}t|ƒ}|| _|| _|d | _|| _|| | _	t
|| ƒ| _t
|| ƒ| | _||| j| j	d  ƒ| _t td|ƒ fd|| j|dd�fgƒ¡| _t t ||d |d  ¡¡| _t tt |d ¡t |d ¡ƒ¡ d¡}	|	dd d …d f |	dd d d …f   ¡ }
|
d |d  |
d  }
| jd	|
d
d� i | _d S )Nç      à¿r.   ÚactÚlnr   ©r(   r   .Úattention_bias_idxsF©Ú
persistent)r   r   r   rF   r   Úuse_convÚ	num_headsÚscaleÚkey_dimÚkey_attn_dimÚintÚval_dimÚval_attn_dimÚqkvr   Ú
Sequentialr   ÚprojÚ	ParameterrC   ÚzerosÚattention_biasesÚstackr   ÚarangerL   ÚabsÚregister_bufferrr   )r    Údimr   r}   Ú
attn_ratiorh   r|   rb   Úln_layerÚposÚrel_posr)   r+   r,   r   ¨   s*    



þ
 *(zAttention.__init__Tc                    s    t ƒ  |¡ |r| jri | _d S r;   ©r   Útrainrr   ©r    Úmoder)   r+   r,   r”   Ë   s    
zAttention.train©ÚdeviceÚreturnc                 C   s^   t j ¡ s| jr$| jd d …| jf S t|ƒ}|| jkrP| jd d …| jf | j|< | j| S d S r;   ©rC   ÚjitÚ
is_tracingÚtrainingr‰   ry   Ústrrr   ©r    r˜   Z
device_keyr+   r+   r,   Úget_attention_biasesÑ   s    
zAttention.get_attention_biasesc                 C   sX  | j r˜|j\}}}}|  |¡ || jd|| ¡j| j| j| jgdd�\}}}| dd¡| | j	 |  
|j¡ }	|	jdd�}	||	 dd¡  |d||¡}n²|j\}}
}|  |¡ ||
| jd¡j| j| j| jgdd�\}}}| dddd¡}| dddd¡}| dddd¡}|| | j	 |  
|j¡ }	|	jdd�}	|	|  dd¡ ||
| j¡}|  |¡}|S )NrS   r.   ©rŽ   éþÿÿÿrZ   r   r   )r|   r3   r„   rU   r}   Úsplitr   r‚   Ú	transposer~   r    r˜   Úsoftmaxrl   rm   rƒ   r†   )r    r=   rn   rp   ÚHÚWÚqÚkÚvÚattnro   r+   r+   r,   r>   Ú   s@    
   ÿ ÿ"
   ÿ ÿ
zAttention.forward)T©r@   rA   rB   r   rž   rC   ZTensorÚ__annotations__r   ÚSiLUr   rD   r”   r˜   r    r>   rE   r+   r+   r)   r,   rq   ¥   s   
ø#	rq   c                       st   e Zd ZU eeejf ed< ddddddej	f‡ fdd„	Z
e ¡ d‡ fd
d„	ƒZejejdœdd„Zdd„ Z‡  ZS )ÚAttentionDownsamplerr   rW   ç       @r.   rt   Fc              	      sÞ  t ƒ  ¡  t|ƒ}|| _|| _|| _|| _|| | _t|| ƒ| _	| j	| j | _
|d | _|| _| jrŽt}ttj|	rxdnd|	r‚dnddd�}nt}tt||	d�}||| j
| j ƒ| _t td||d	�fd
||| jƒfgƒ¡| _t td|
ƒ fd
|| j
|ƒfgƒ¡| _t t ||d |d  ¡¡| _t tt |d ¡t |d ¡ƒ¡ d¡}t ttjd|d |d�tjd|d |d�ƒ¡ d¡}|dd d …d f |dd d d …f    ¡ }|d |d  |d  }| j!d|dd� i | _"d S )Nru   rZ   r   r   F)r#   r%   rg   )rh   rk   Zdown)r$   rw   rv   )Ústep.ry   rz   )#r   r   r   r$   rh   r}   r   r€   r�   r‚   rƒ   r~   r|   r   r   r   ri   rF   rf   Úkvr…   r   r¨   r†   r‡   rC   rˆ   r‰   rŠ   r   r‹   rL   rŒ   r�   rr   )r    Úin_dimÚout_dimr   r}   r�   r$   rh   r|   rk   rb   r�   Z	sub_layerZk_posZq_posr’   r)   r+   r,   r   ÷   sT    



 
 þþ
þ
 *þý(zAttentionDownsample.__init__Tc                    s    t ƒ  |¡ |r| jri | _d S r;   r“   r•   r)   r+   r,   r”   0  s    
zAttentionDownsample.trainr—   c                 C   s^   t j ¡ s| jr$| jd d …| jf S t|ƒ}|| jkrP| jd d …| jf | j|< | j| S d S r;   rš   rŸ   r+   r+   r,   r    6  s    
z(AttentionDownsample.get_attention_biasesc                 C   s¤  | j rÔ|j\}}}}|d | j d |d | j d  }}|  |¡ || jd|| ¡j| j| jgdd�\}}	|  	|¡ || j| jd¡}
|
 
dd¡| | j |  |j¡ }|jdd�}|	| 
dd¡  || j||¡}nÂ|j\}}}|  |¡ ||| jd¡j| j| jgdd�\}}	| dddd¡}|	 dddd¡}	|  	|¡ |d| j| j¡ dddd¡}
|
| | j |  |j¡ }|jdd�}||	  
dd¡ |d| j¡}|  |¡}|S )Nr   rS   r.   r¡   r¢   rZ   r   )r|   r3   r$   r²   rU   r}   r£   r   r‚   r¨   r¤   r~   r    r˜   r¥   rm   rƒ   rl   r†   )r    r=   rn   rp   r¦   r§   ZHHZWWr©   rª   r¨   r«   ro   r+   r+   r,   r>   ?  s$    &2" .&
zAttentionDownsample.forward)Tr¬   r+   r+   r)   r,   r¯   ô   s   
õ9	r¯   c                       s6   e Zd ZdZdddejdf‡ fdd„	Zdd„ Z‡  ZS )	ÚLevitMlpzL MLP for Levit w/ normalization + ability to switch btw conv and linear
    NFrO   c                    sZ   t ƒ  ¡  |p|}|p|}|r"tnt}|||ƒ| _|ƒ | _t |¡| _|||dd�| _	d S )Nr   rx   )
r   r   r   rF   Úln1rv   r   rQ   rR   Úln2)r    rI   Zhidden_featuresrJ   r|   rb   rR   r�   r)   r+   r,   r   \  s    	
zLevitMlp.__init__c                 C   s,   |   |¡}|  |¡}|  |¡}|  |¡}|S r;   )r¶   rv   rR   r·   r<   r+   r+   r,   r>   o  s
    



zLevitMlp.forward)	r@   rA   rB   Ú__doc__r   r®   r   r>   rE   r+   r+   r)   r,   rµ   Y  s   ùrµ   c                	       s:   e Zd Zdddejdddddf	‡ fdd	„	Zd
d„ Z‡  ZS )ÚLevitDownsamplerW   rs   r°   Nrt   FrO   c                    sf   t ƒ  ¡  |p|}t|||||||	|
|d�	| _t|t|| ƒ|
|d�| _|dkrXt|ƒnt 	¡ | _
d S )N)	r³   r´   r   r}   r�   rb   rh   r|   rk   ©r|   rb   rO   )r   r   r¯   Úattn_downsamplerµ   r�   Úmlpr
   r   ÚIdentityÚ	drop_path)r    r³   r´   r   r}   r�   Ú	mlp_ratiorb   Úattn_act_layerrh   r|   rk   r¾   r)   r+   r,   r   x  s(    
÷
üzLevitDownsample.__init__c                 C   s"   |   |¡}||  |  |¡¡ }|S r;   )r»   r¾   r¼   r<   r+   r+   r,   r>   ž  s    
zLevitDownsample.forward©r@   rA   rB   r   r®   r   r>   rE   r+   r+   r)   r,   r¹   w  s   ó&r¹   c                       s8   e Zd Zdddddejddf‡ fdd	„	Zd
d„ Z‡  ZS )Ú
LevitBlockrW   rs   r°   rt   FNrO   c              	      s|   t ƒ  ¡  |	p|}	t|||||||	d�| _|
dkr:t|
ƒnt ¡ | _t|t	|| ƒ||d�| _
|
dkrnt|
ƒnt ¡ | _d S )N)rŽ   r   r}   r�   rh   r|   rb   rO   rº   )r   r   rq   r«   r
   r   r½   Ú
drop_path1rµ   r�   r¼   Ú
drop_path2)r    rŽ   r   r}   r�   r¿   rh   r|   rb   rÀ   r¾   r)   r+   r,   r   ¥  s&    
ù	
üzLevitBlock.__init__c                 C   s,   ||   |  |¡¡ }||  |  |¡¡ }|S r;   )rÃ   r«   rÄ   r¼   r<   r+   r+   r,   r>   È  s    zLevitBlock.forwardrÁ   r+   r+   r)   r,   rÂ   ¤  s   õ#rÂ   c                
       s<   e Zd Zddddejdddddf
‡ fd	d
„	Zdd„ Z‡  ZS )Ú
LevitStagerY   rW   rs   Nrt   Ú FrO   c                    s¤   t ƒ  ¡  t|
ƒ}
|rJt||||| dd||	|
||d�| _dd„ |
D ƒ}
n||ksVt‚t ¡ | _g }t|ƒD ]&}|t	|||||||	|
||d�
g7 }qltj
|Ž | _d S )Nrs   r°   )	r   r}   r�   r¿   rb   rÀ   rh   r|   r¾   c                 S   s   g | ]}|d  d d  ‘qS )r   r.   r+   ©Ú.0Úrr+   r+   r,   Ú
<listcomp>ð  s     z'LevitStage.__init__.<locals>.<listcomp>)r}   r�   r¿   rb   rÀ   rh   r|   r¾   )r   r   r   r¹   Ú
downsampleÚAssertionErrorr   r½   ÚrangerÂ   r…   Úblocks)r    r³   r´   r   Údepthr}   r�   r¿   rb   rÀ   rh   rË   r|   r¾   rÎ   Ú_r)   r+   r,   r   Ï  sD    
õ
özLevitStage.__init__c                 C   s   |   |¡}|  |¡}|S r;   )rË   rÎ   r<   r+   r+   r,   r>     s    

zLevitStage.forwardrÁ   r+   r+   r)   r,   rÅ   Î  s   ò6rÅ   c                       s�   e Zd ZdZd%‡ fdd„	Zejjdd„ ƒZejjd&dd„ƒZ	ejjd'dd„ƒZ
ejjdd„ ƒZd(dd„Zdd„ Zd)ed œd!d"„Zd#d$„ Z‡  ZS )*r   zß Vision Transformer with support for patch or hybrid CNN input stage

    NOTE: distillation is defaulted to True since pretrained weights use it, will cause problems
    w/ train scripts that don't take tuple outputs,
    éà   rZ   éè  ©éÀ   é@   ©é   ©rZ   r°   NÚs16Ú	subsampleÚ
hard_swishFÚavgrO   c                    s  t ƒ  ¡  t|ƒ}t|p|ƒ}|| _|| _|| _|d | _|| _|| _d| _	g | _
t|ƒ}t|ƒ|ksjt‚t|ƒ|ƒ}t|ƒ|ƒ}t|ƒ|	ƒ}	|
d k	r®|dks¢t‚|
| _|}nF|dksºt‚|dkrØt||d |d�| _nt||d |d�| _| jj}tdd	„ tt|ƒt|ƒƒD ƒƒ}|d }g }t|ƒD ]¬}|dk�r:dnd
‰ |t||| ||| || || |	| ||||ˆ dk�rz|nd|d�g7 }|ˆ 9 }t‡ fdd	„|D ƒƒ}|  j
t|| |d|› �d�g7  _
|| }�q(tj|Ž | _|dk�rþt|d ||d�nt ¡ | _d S )NrS   Fr.   )rÙ   Ús8rÙ   r   )rb   c                 S   s   g | ]\}}|| ‘qS r+   r+   )rÈ   ÚiÚpr+   r+   r,   rÊ   D  s     z"Levit.__init__.<locals>.<listcomp>r   rÆ   )
rÏ   r}   r�   r¿   rb   rÀ   rh   r|   rË   r¾   c                    s   g | ]}|d  ˆ  d  ‘qS )r   r+   rÇ   ©Zstage_strider+   r,   rÊ   Z  s     zstages.)Znum_chsZ	reductionÚmodule©rR   )r   r   r	   r|   Únum_classesÚglobal_poolÚnum_featuresÚ	embed_dimÚ	drop_rateÚgrad_checkpointingZfeature_infoÚlenrÌ   r   Ústemrd   rV   r$   ÚtupleÚzipr   rÍ   rÅ   Údictr   r…   ÚstagesrM   r½   Úhead)r    Zimg_sizeZin_chansrã   ræ   r   rÏ   r}   r�   r¿   Zstem_backboneZstem_strideÚ	stem_typeZdown_oprb   rÀ   r|   rä   rç   Zdrop_path_rateZ
num_stagesr$   rh   r³   rî   rÞ   r)   rà   r,   r     sf    

 ó
$zLevit.__init__c                 C   s   dd„ |   ¡  ¡ D ƒS )Nc                 S   s   h | ]}d |kr|’qS )r‰   r+   )rÈ   r=   r+   r+   r,   Ú	<setcomp>d  s      z(Levit.no_weight_decay.<locals>.<setcomp>)Ú
state_dictÚkeys©r    r+   r+   r,   Úno_weight_decayb  s    zLevit.no_weight_decayc                 C   s   t dddgd�}|S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )rê   rÎ   )rí   )r    ZcoarseZmatcherr+   r+   r,   Úgroup_matcherf  s
    þzLevit.group_matcherTc                 C   s
   || _ d S r;   )rè   ©r    Úenabler+   r+   r,   Úset_grad_checkpointingn  s    zLevit.set_grad_checkpointingc                 C   s   | j S r;   )rï   rô   r+   r+   r,   Úget_classifierr  s    zLevit.get_classifierc                 C   s@   || _ |d k	r|| _|dkr2t| jd || jd�nt ¡ | _d S )Nr   rS   râ   )rã   rä   rM   ræ   rç   r   r½   rï   ©r    rã   rä   Zdistillationr+   r+   r,   Úreset_classifierv  s    ÿ  ÿÿzLevit.reset_classifierc                 C   sN   |   |¡}| js"| d¡ dd¡}| jr@tj ¡ s@t| j	|ƒ}n
|  	|¡}|S )Nr.   r   )
rê   r|   rL   r¤   rè   rC   r›   Úis_scriptingr   rî   r<   r+   r+   r,   Úforward_features}  s    

zLevit.forward_features©Ú
pre_logitsc                 C   s:   | j dkr(| jr|jdd�n
|jdd�}|r0|S |  |¡S )NrÜ   ©r¢   rS   r¡   r   )rä   r|   Úmeanrï   )r    r=   r   r+   r+   r,   Úforward_head‡  s    
zLevit.forward_headc                 C   s   |   |¡}|  |¡}|S r;   )rþ   r  r<   r+   r+   r,   r>   Œ  s    

zLevit.forward)rÑ   rZ   rÒ   rÓ   rÕ   rÖ   rØ   r°   r°   NNrÙ   rÚ   rÛ   NFrÜ   rO   rO   )F)T)NN)F)r@   rA   rB   r¸   r   rC   r›   Úignorerõ   rö   rù   rú   rü   rþ   Úboolr  r>   rE   r+   r+   r)   r,   r     sB                      ìP



c                       sX   e Zd Z‡ fdd„Zejjdd„ ƒZddd„Zejjdd	d
„ƒZ	de
dœdd„Z‡  ZS )ÚLevitDistilledc                    s:   t ƒ j||Ž | jdkr&t| j| jƒnt ¡ | _d| _d S )Nr   F)	r   r   rã   rM   rå   r   r½   Ú	head_distÚdistilled_training)r    ÚargsÚkwargsr)   r+   r,   r   “  s    "zLevitDistilled.__init__c                 C   s   | j | jfS r;   )rï   r  rô   r+   r+   r,   rú   ˜  s    zLevitDistilled.get_classifierNc                 C   sZ   || _ |d k	r|| _|dkr.t| j|| jd�nt ¡ | _|dkrLt| j|ƒnt ¡ | _d S )Nr   râ   )	rã   rä   rM   rå   rç   r   r½   rï   r  rû   r+   r+   r,   rü   œ  s    ÿ  ÿÿzLevitDistilled.reset_classifierTc                 C   s
   || _ d S r;   )r  r÷   r+   r+   r,   Úset_distilled_training¤  s    z%LevitDistilled.set_distilled_trainingFrÿ   c                 C   st   | j dkr(| jr|jdd�n
|jdd�}|r0|S |  |¡|  |¡ }}| jrd| jrdtj 	¡ sd||fS || d S d S )NrÜ   r  r¡   r   r.   )
rä   r|   r  rï   r  r  r�   rC   r›   rý   )r    r=   r   Zx_distr+   r+   r,   r  ¨  s    
zLevitDistilled.forward_head)NN)T)F)r@   rA   rB   r   rC   r›   r  rú   rü   r  r  r  rE   r+   r+   r)   r,   r  ’  s   

r  c                 C   s²   d| kr| d } dd„ |   ¡ D ƒ} | ¡ }i }t| ¡ |  ¡ | ¡ |  ¡ ƒD ]`\}}}}|jdkr„|jdkr„|d d …d d …d d f }|j|jkr¤d|ks¤d|ks¤t‚|||< qL|S )NÚmodelc                 S   s   i | ]\}}d |kr||“qS )ry   r+   )rÈ   r©   rª   r+   r+   r,   Ú
<dictcomp>»  s       z(checkpoint_filter_fn.<locals>.<dictcomp>rY   r.   rï   ústem.conv1.linear)Úitemsrò   rì   ró   ÚvaluesÚndimr3   rÌ   )rò   r  ÚDZout_dictÚkaÚkbÚvaZvbr+   r+   r,   Úcheckpoint_filter_fn¶  s    *
r  )é€   é   é€  re   )rY   é   rW   )r.   rZ   rY   )ræ   r   r}   rÏ   )rY   rW   r×   )rY   rY   rY   )rÔ   i   r  é    )rZ   é   r  )r  r  é   )r  r  é   )r  é	   r×   ZsilurÝ   )ræ   r   r}   rÏ   rb   rð   )r  é€  i€  rÕ   )rW   é
   rt   )r  r  i   )rW   r×   re   )ræ   r   r}   rÏ   rb   )rY   rW   r  )r  r   r  )rW   r!  r×   )
Ú
levit_128sÚ	levit_128Ú	levit_192Ú	levit_256Ú	levit_384Úlevit_384_s8Úlevit_512_s8Ú	levit_512Ú
levit_256dÚ
levit_512dFTc           	      K   s�   d| k}|  dd¡}| dd ¡r,|s,tdƒ‚|d krR| tkrB| }n|rR|  dd¡}tt| f|Ž}t|rltnt| |ft	td|d�d	œ|—Ž}|S )
NZ_convÚout_indices)r   r   r.   Zfeatures_onlyzBfeatures_only not implemented for LeVit in non-convolutional mode.rÆ   T)Zflatten_sequentialr,  )Zpretrained_filter_fnZfeature_cfg)
ÚpopÚgetÚRuntimeErrorÚ
model_cfgsÚreplacerí   r   r  r   r  )	ÚvariantZcfg_variantÚ
pretrainedÚ	distilledr
  Zis_convr,  Z	model_cfgr  r+   r+   r,   Úcreate_levitê  s,    
ý
ûúr5  rÆ   c                 K   s    | ddd dddt tdddœ|–S )	NrÒ   )rZ   rÑ   rÑ   gÍÌÌÌÌÌì?ZbicubicTr  )úhead.linearzhead_dist.linear)Úurlrã   Z
input_sizeÚ	pool_sizeZcrop_pctÚinterpolationZfixed_input_sizer  rP   Z
first_convÚ
classifier)r   r   )r7  r
  r+   r+   r,   Ú_cfg  s          ûúr;  ztimm/)Ú	hf_hub_id)rY   rY   )r<  r8  r6  )r:  )zlevit_128s.fb_dist_in1kzlevit_128.fb_dist_in1kzlevit_192.fb_dist_in1kzlevit_256.fb_dist_in1kzlevit_384.fb_dist_in1kzlevit_conv_128s.fb_dist_in1kzlevit_conv_128.fb_dist_in1kzlevit_conv_192.fb_dist_in1kzlevit_conv_256.fb_dist_in1kzlevit_conv_384.fb_dist_in1kzlevit_384_s8.untrainedzlevit_512_s8.untrainedzlevit_512.untrainedzlevit_256d.untrainedzlevit_512d.untrainedzlevit_conv_384_s8.untrainedzlevit_conv_512_s8.untrainedzlevit_conv_512.untrainedzlevit_conv_256d.untrainedzlevit_conv_512d.untrained)r™   c                 K   s   t dd| i|—ŽS )Nr"  r3  )r"  ©r5  ©r3  r
  r+   r+   r,   r"  C  s    r"  c                 K   s   t dd| i|—ŽS )Nr#  r3  )r#  r=  r>  r+   r+   r,   r#  H  s    r#  c                 K   s   t dd| i|—ŽS )Nr$  r3  )r$  r=  r>  r+   r+   r,   r$  M  s    r$  c                 K   s   t dd| i|—ŽS )Nr%  r3  )r%  r=  r>  r+   r+   r,   r%  R  s    r%  c                 K   s   t dd| i|—ŽS )Nr&  r3  )r&  r=  r>  r+   r+   r,   r&  W  s    r&  c                 K   s   t dd| i|—ŽS )Nr'  r3  )r'  r=  r>  r+   r+   r,   r'  \  s    r'  c                 K   s   t d| ddœ|—ŽS )Nr(  F©r3  r4  )r(  r=  r>  r+   r+   r,   r(  a  s    r(  c                 K   s   t d| ddœ|—ŽS )Nr)  Fr?  )r)  r=  r>  r+   r+   r,   r)  f  s    r)  c                 K   s   t d| ddœ|—ŽS )Nr*  Fr?  )r*  r=  r>  r+   r+   r,   r*  k  s    r*  c                 K   s   t d| ddœ|—ŽS )Nr+  Fr?  )r+  r=  r>  r+   r+   r,   r+  p  s    r+  c                 K   s   t d| ddœ|—ŽS )NÚlevit_conv_128sT©r3  r|   )r@  r=  r>  r+   r+   r,   r@  u  s    r@  c                 K   s   t d| ddœ|—ŽS )NÚlevit_conv_128TrA  )rB  r=  r>  r+   r+   r,   rB  z  s    rB  c                 K   s   t d| ddœ|—ŽS )NÚlevit_conv_192TrA  )rC  r=  r>  r+   r+   r,   rC    s    rC  c                 K   s   t d| ddœ|—ŽS )NÚlevit_conv_256TrA  )rD  r=  r>  r+   r+   r,   rD  „  s    rD  c                 K   s   t d| ddœ|—ŽS )NÚlevit_conv_384TrA  )rE  r=  r>  r+   r+   r,   rE  ‰  s    rE  c                 K   s   t d| ddœ|—ŽS )NÚlevit_conv_384_s8TrA  )rF  r=  r>  r+   r+   r,   rF  Ž  s    rF  c                 K   s   t d| dddœ|—ŽS )NÚlevit_conv_512_s8TF©r3  r|   r4  )rG  r=  r>  r+   r+   r,   rG  “  s    rG  c                 K   s   t d| dddœ|—ŽS )NÚlevit_conv_512TFrH  )rI  r=  r>  r+   r+   r,   rI  ˜  s    rI  c                 K   s   t d| dddœ|—ŽS )NÚlevit_conv_256dTFrH  )rJ  r=  r>  r+   r+   r,   rJ  �  s    rJ  c                 K   s   t d| dddœ|—ŽS )NÚlevit_conv_512dTFrH  )rK  r=  r>  r+   r+   r,   rK  ¢  s    rK  )NFT)rÆ   )F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)Fr¸   Úcollectionsr   Ú	functoolsr   Útypingr   rC   Ztorch.nnr   Z	timm.datar   r   Ztimm.layersr   r   r	   r
   r   r   Z_builderr   Z_manipulater   Ú	_registryr   r   Ú__all__ÚModuler   rF   rM   r…   rV   rd   rf   rq   r¯   rµ   r¹   rÂ   rÅ   r   r  r  rí   r0  r5  r;  Zdefault_cfgsr"  r#  r$  r%  r&  r'  r(  r)  r*  r+  r@  rB  rC  rD  rE  rF  rG  rI  rJ  rK  r+   r+   r+   r,   Ú<module>   sr   Oe-*= $   ÿ   ÿ   ÿ   ÿ   ÿ    þ    þ    ÿ    ÿ    ÿå 

ÿÿÿÿÿþþþþþÎ6