U
    ¯Ê¹j¶X  ã                   @   s"  d Z ddlmZ ddlZddlm  mZ ddlmZ ddlm	Z	m
Z
 ddlmZmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ d6dd„ZG dd„ dejƒZd7dd„ZG dd„ dejƒZ G dd„ dejƒZ!G dd„ dejƒZ"G dd„ dejƒZ#G dd„ dejƒZ$G dd„ dejƒZ%G dd „ d ejƒZ&d!d"„ Z'd8d$d%„Z(d9d'd(„Z)ee)d)d*�e)d)d*�e)d)d*�e)d)d+d,d-d.�e)d)d+d,d-d.�e)d)d+d,d-d.�e)d)d*�e)d)d*�e)d)d*�e)d)d+d,d-d.�e)d)d+d,d-d.�e)d)d+d,d-d.�d/œƒZ*ed:d0d1„ƒZ+ed;d2d3„ƒZ,ed<d4d5„ƒZ-dS )=zª Next-ViT

As described in https://arxiv.org/abs/2207.05501

Next-ViT model defs and weights adapted from https://github.com/bytedance/Next-ViT, original copyright below
é    )ÚpartialN)Únn©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚtrunc_normal_ÚConvMlpÚget_norm_layerÚget_act_layerÚuse_fused_attn)ÚClassifierHeadé   )Úbuild_model_with_cfg)Úregister_notrace_function)Úcheckpoint_seq)Úgenerate_default_cfgsÚregister_modelc                 C   s4  | j j}| jdkr:tj| j|jd� | ¡ ¡}t 	|¡| _| jj}|dkr¤|j
dks\tdƒ‚|jdksntdƒ‚|j |j¡ d¡}||j  }|j|j |j |  }n°|j
dks¶tdƒ‚|jdksÈtdƒ‚|j
dksÚtdƒ‚|jdksìtdƒ‚|j |j¡ d¡}	|j |j¡ d¡}
|	|j  |
 |j  }|
|j  |j|j |j |	  |j  |j }t| tjƒ�rŠ|| }| | d| d¡¡ |¡¡ nŒt| tjƒ�r|jd dk�r¸|jd	 dk�s¼t‚| |jd
 |jd ¡}|| }| | d| d¡¡ |¡¡ | |jd
 |jd dd¡}| |¡ || j _|| j_dS )z/ Merge pre BN to reduce inference runtime.
    N)ÚdeviceTz2Unsupported bn_module.track_running_stats is Falsez%Unsupported bn_module.affine is Falseç      à¿r   é   é   r   )ÚweightÚdataÚbiasÚtorchÚzerosÚout_chsr   Útyper   Ú	ParameterZtrack_running_statsÚAssertionErrorZaffineZrunning_varÚaddÚepsÚpowZrunning_meanÚ
isinstanceÚLinearZmul_ÚviewÚsizeZ	expand_asÚConv2dÚshapeÚreshapeZadd_)ÚmoduleZpre_bn_1Zpre_bn_2r   r   r   Zscale_invstdZextra_weightZ
extra_biasZscale_invstd_1Zscale_invstd_2© r,   úR/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/nextvit.pyÚmerge_pre_bn   sH    

ÿþÿ $
r.   c                       s4   e Zd Zdddejejf‡ fdd„	Zdd„ Z‡  ZS )ÚConvNormActr   r   c              	      s>   t t| ƒ ¡  tj||||d|dd�| _||ƒ| _|ƒ | _d S )Nr   F©Úkernel_sizeÚstrideÚpaddingÚgroupsr   )Úsuperr/   Ú__init__r   r(   ÚconvÚnormÚact)ÚselfÚin_chsr   r1   r2   r4   Ú
norm_layerÚ	act_layer©Ú	__class__r,   r-   r6   G   s    
     þ
zConvNormAct.__init__c                 C   s"   |   |¡}|  |¡}|  |¡}|S ©N)r7   r8   r9   ©r:   Úxr,   r,   r-   ÚforwardX   s    


zConvNormAct.forward)	Ú__name__Ú
__module__Ú__qualname__r   ÚBatchNorm2dÚReLUr6   rC   Ú__classcell__r,   r,   r>   r-   r/   F   s   ør/   c                 C   sB   |d kr|}t |t| |d  ƒ| | ƒ}|d|  k r>||7 }|S )Nr   gÍÌÌÌÌÌì?)ÚmaxÚint)ÚvZdivisorZ	min_valueÚnew_vr,   r,   r-   Ú_make_divisible_   s    rN   c                       s,   e Zd Zdejf‡ fdd„	Zdd„ Z‡  ZS )Ú
PatchEmbedr   c                    s¢   t t| ƒ ¡  |dkrLtjddddd�| _tj||dddd�| _||ƒ| _nR||kr€t 	¡ | _tj||dddd�| _||ƒ| _nt 	¡ | _t 	¡ | _t 	¡ | _d S )Nr   )r   r   TF)r2   Z	ceil_modeZcount_include_padr   )r1   r2   r   )
r5   rO   r6   r   Z	AvgPool2dÚpoolr(   r7   r8   ÚIdentity)r:   r;   r   r2   r<   r>   r,   r-   r6   j   s    


zPatchEmbed.__init__c                 C   s   |   |  |  |¡¡¡S r@   )r8   r7   rP   rA   r,   r,   r-   rC      s    zPatchEmbed.forward)rD   rE   rF   r   rG   r6   rC   rI   r,   r,   r>   r-   rO   i   s   ürO   c                       s2   e Zd ZdZejejf‡ fdd„	Zdd„ Z‡  Z	S )ÚConvAttentionz,
    Multi-Head Convolutional Attention
    c              	      sV   t t| ƒ ¡  tj||ddd|| dd�| _||ƒ| _|ƒ | _tj||ddd�| _d S )Nr   r   Fr0   )r1   r   )	r5   rR   r6   r   r(   Úgroup_conv3x3r8   r9   Ú
projection)r:   r   Úhead_dimr<   r=   r>   r,   r-   r6   ˆ   s         þ
zConvAttention.__init__c                 C   s,   |   |¡}|  |¡}|  |¡}|  |¡}|S r@   )rS   r8   r9   rT   ©r:   rB   Úoutr,   r,   r-   rC   ’   s
    



zConvAttention.forward)
rD   rE   rF   Ú__doc__r   rG   rH   r6   rC   rI   r,   r,   r>   r-   rR   ƒ   s   
rR   c                       sL   e Zd ZdZdddddejejf‡ fdd„	Ze 	¡ dd	„ ƒZ
d
d„ Z‡  ZS )ÚNextConvBlockz 
    Next Convolution Block
    r   ç        é    g      @c
           
         s’   t t| ƒ ¡  || _|| _|| dks*t‚t||||d�| _t||||	d�| _	t
|ƒ| _||ƒ| _t|t|| ƒ|d|	d�| _t
|ƒ| _d| _d S )Nr   )r<   )r<   r=   T)Úhidden_featuresÚdropr   r=   F)r5   rY   r6   r;   r   r    rO   Úpatch_embedrR   Úmhcar   Úattn_drop_pathr8   r	   rK   ÚmlpÚmlp_drop_pathÚis_fused)
r:   r;   r   r2   Ú	drop_pathr]   rU   Ú	mlp_ratior<   r=   r>   r,   r-   r6   ž   s,    ü


û
zNextConvBlock.__init__c                 C   s&   | j s"t| jj| jƒ d | _d| _ d S ©NT)rc   r.   ra   Úfc1r8   ©r:   r,   r,   r-   ÚreparameterizeÃ   s    zNextConvBlock.reparameterizec                 C   s@   |   |¡}||  |  |¡¡ }|  |¡}||  |  |¡¡ }|S r@   )r^   r`   r_   r8   rb   ra   rV   r,   r,   r-   rC   Ê   s
    

zNextConvBlock.forward©rD   rE   rF   rX   r   rG   rH   r6   r   Zno_gradri   rC   rI   r,   r,   r>   r-   rY   ™   s   ö%
rY   c                       sL   e Zd ZU dZejje ed< dddddde	j
f‡ fdd	„	Zd
d„ Z‡  ZS )ÚEfficientAttentionz-
    Efficient Multi-Head Self Attention
    Ú
fused_attnNr[   TrZ   r   c	           	         sô   t ƒ  ¡  || _|d k	r|n|| _| j| | _|| _|d | _tƒ | _t	j
|| j|d�| _t	j
|| j|d�| _t	j
|| j|d�| _t	 
| j| j¡| _t	 |¡| _t	 |¡| _|| _|d | _|dkrät	j| j| jd�| _||ƒ| _nd | _d | _d S )Nr   )r   r   r   )r1   r2   )r5   r6   ÚdimÚout_dimÚ	num_headsrU   Úscaler   rl   r   r%   ÚqÚkrL   ÚprojZDropoutÚ	attn_dropÚ	proj_dropÚsr_ratioZN_ratioZ	AvgPool1dÚsrr8   )	r:   rm   rn   rU   Zqkv_biasrt   ru   rv   r<   r>   r,   r-   r6   Ù   s(    


zEfficientAttention.__init__c           	      C   s2  |j \}}}|  |¡ ||| j| j¡ dddd¡}| jd k	r`|  | dd¡¡}|  |¡ dd¡}|  	|¡ |d| j| j¡ dd¡}|  
|¡ |d| j| j¡ dd¡}| jrÌtj|||| jrÂ| jjndd�}n8|| j }|| dd¡ }|jdd	�}|  |¡}|| }| dd¡ |||¡}|  |¡}|  |¡}|S )
Nr   r   r   r   éÿÿÿÿrZ   )Z	dropout_péþÿÿÿ©rm   )r)   rq   r*   ro   rU   Zpermuterw   Ú	transposer8   rr   rL   rl   ÚFZscaled_dot_product_attentionZtrainingrt   Úprp   Zsoftmaxrs   ru   )	r:   rB   ÚBÚNÚCrq   rr   rL   Zattnr,   r,   r-   rC   ü   s.    &
""  þ



zEfficientAttention.forward)rD   rE   rF   rX   r   ÚjitÚFinalÚboolÚ__annotations__r   ZBatchNorm1dr6   rC   rI   r,   r,   r>   r-   rk   Ó   s   
÷#rk   c                	       sP   e Zd ZdZdddddddejejf	‡ fdd„	Ze 	¡ d	d
„ ƒZ
dd„ Z‡  ZS )ÚNextTransformerBlockz 
    Next Transformer Block
    r   r   r[   ç      è?rZ   c                    sô   t t| ƒ ¡  || _|| _|| _tt|| ƒdƒ| _|| j | _	t
|| j|ƒ| _|| jƒ| _t| j|||	|
d�| _t|| ƒ| _t
| j| j	d|d�| _t| j	|||d�| _t|d|  ƒ| _||ƒ| _t|t|| ƒ||
d�| _t|ƒ| _d| _d S )Nr[   )rU   rv   rt   ru   r   )r2   r<   )rU   r<   r=   )r\   r=   r]   F)r5   r…   r6   r;   r   Úmix_block_ratiorN   rK   Zmhsa_out_chsZmhca_out_chsrO   r^   Únorm1rk   Úe_mhsar   Úmhsa_drop_pathrT   rR   r_   Úmhca_drop_pathÚnorm2r	   ra   rb   rc   )r:   r;   r   rd   r2   rv   re   rU   r‡   rt   r]   r<   r=   r>   r,   r-   r6     sB    ûü

ü
zNextTransformerBlock.__init__c                 C   sª   | j s¦t| jj| jƒ | jjd k	r\t| jj| j| jjƒ t| jj| j| jjƒ t 	¡ | j_n t| jj| jƒ t| jj| jƒ t 	¡ | _t| j
j| jƒ t 	¡ | _d| _ d S rf   )rc   r.   r‰   rq   rˆ   r8   rr   rL   r   rQ   ra   rg   rŒ   rh   r,   r,   r-   ri   S  s    

z#NextTransformerBlock.reparameterizec                 C   s¶   |   |¡}|j\}}}}|  |¡}| ||d¡ dd¡}|  |  |¡¡}|| dd¡ ||||¡ }|  |¡}||  |  	|¡¡ }t
j||gdd�}|  |¡}||  |  |¡¡ }|S )Nrx   ry   r   rz   )r^   r)   rˆ   r*   r{   rŠ   r‰   rT   r‹   r_   r   ÚcatrŒ   rb   ra   )r:   rB   r~   r€   ÚHÚWrW   r,   r,   r-   rC   d  s    



zNextTransformerBlock.forwardrj   r,   r,   r>   r-   r…     s   	ó5
r…   c                	       sN   e Zd Zdddddddejejf	‡ fdd„	Zejj	dd	d
„ƒZ
dd„ Z‡  ZS )Ú	NextStager   r   ç      ð?rZ   r[   c                    sÐ   t ƒ  ¡  d| _g }t|ƒD ]¢\}}|dkr0|nd}|| }|| }t|	ttfƒrZ|	| n|	}|tkrŠt||||||
||d�}| |¡ n0|t	krºt	||||||
|||||d�}| |¡ |}qt
j|Ž | _d S )NFr   r   )r2   rd   r]   rU   r<   r=   )	rd   r2   rv   rU   r‡   rt   r]   r<   r=   )r5   r6   Úgrad_checkpointingÚ	enumerater$   ÚlistÚtuplerY   Úappendr…   r   Ú
SequentialÚblocks)r:   r;   Ú	block_chsÚblock_typesr2   rv   r‡   r]   rt   rd   rU   r<   r=   r˜   Z	block_idxZ
block_typer   ÚdprÚlayerr>   r,   r-   r6   x  sJ    
ø
õ
zNextStage.__init__Tc                 C   s
   || _ d S r@   )r’   )r:   Úenabler,   r,   r-   Úset_grad_checkpointing¯  s    z NextStage.set_grad_checkpointingc                 C   s,   | j rtj ¡ st| j|ƒ}n
|  |¡}|S r@   )r’   r   r�   Úis_scriptingr   r˜   rA   r,   r,   r-   rC   ³  s    
zNextStage.forward)T)rD   rE   rF   r   rG   rH   r6   r   r�   Úignorerž   rC   rI   r,   r,   r>   r-   r�   v  s   ó7r�   c                       s    e Zd Zdddddddddd	d
ejdf‡ fdd„	Zdd„ Zejj	d!dd„ƒZ
ejj	d"dd„ƒZejj	dd„ ƒZd#dd„Zdd„ Zd$edœdd„Zdd „ Z‡  ZS )%ÚNextViTéè  Úavg)é@   r[   r¤   ©r   é   é
   r   )r   r   r   r   )é   r¦   r   r   çš™™™™™¹?rZ   r[   r†   Nc                    sp  t t| ƒ ¡  d| _|| _t|ƒ}|d kr:ttjdd�}nt	|ƒ}dgˆ d  dgˆ d d  dg d	d	d	d	d
gˆ d d  dgˆ d d  dg g| _
dd„ t| j
ƒD ƒ| _tgˆ d  tgˆ d d  tg tttttgˆ d d  tgˆ d d  tg g| _t t||d dd||d�t|d |d dd||d�t|d |d dd||d�t|d |d dd||d�¡| _|d  }}g }d}dd„ t d|tˆ ƒ¡ ˆ ¡D ƒ}ttˆ ƒƒD ]f}t|| j
| | j| || || |||
|	|| ||d�}| j
| d  }}||g7 }|ˆ | 7 }�q´|| _tj|Ž | _||ƒ| _t|||d�| _‡ fdd„ttˆ ƒƒD ƒ| _|   ¡  d S )NFT)Zinplaceé`   r   éÀ   r   é   é€  i   r   é   i   r   i   c                 S   s0   g | ](\}}t |d  d|d  d|› �d�‘qS )rx   r   zstages.)Znum_chsZ	reductionr+   ©Údict)Ú.0ÚiÚscr,   r,   r-   Ú
<listcomp>Ü  s   ü
ýz$NextViT.__init__.<locals>.<listcomp>)r1   r2   r<   r=   rx   c                 S   s   g | ]}|  ¡ ‘qS r,   )Útolist)r±   rB   r,   r,   r-   r´   ò  s     )r;   r™   rš   r2   rv   r‡   rU   r]   rt   rd   r<   r=   )Ú	pool_typeZin_featuresÚnum_classesc                    s$   g | ]}t ˆ d |d … ƒd ‘qS )Nr   )Úsum)r±   Úidx©Údepthsr,   r-   r´   
  s     )!r5   r¡   r6   r’   r·   r
   r   r   rH   r   Zstage_out_chsr“   Zfeature_inforY   r…   Zstage_block_typesr—   r/   Ústemr   Zlinspacer¸   ÚsplitÚrangeÚlenr�   Znum_featuresÚstagesr8   r   ÚheadZstage_out_idxÚ_initialize_weights)r:   Zin_chansr·   Úglobal_poolZstem_chsr»   ÚstridesZ	sr_ratiosÚdrop_path_rateZattn_drop_rateZ	drop_raterU   r‡   r<   r=   r;   r   rÀ   r¹   r›   Z	stage_idxÚstager>   rº   r-   r6   ¼  sl    üüüü"ô

zNextViT.__init__c                 C   s”   |   ¡ D ]†\}}t|tjƒrPt|jdd� t|dƒrŽ|jd k	rŽtj 	|jd¡ qt|tj
ƒrt|jdd� t|dƒr|jd k	rtj 	|jd¡ qd S )Ng{®Gáz”?)Ústdr   r   )Znamed_modulesr$   r   r%   r   r   Úhasattrr   ÚinitZ	constant_r(   )r:   ÚnÚmr,   r,   r-   rÂ     s    zNextViT._initialize_weightsFc                 C   s   t d|rdnddgd�S )Nz^stemz^stages\.(\d+))z^stages\.(\d+)\.blocks\.(\d+)N)z^norm)iŸ† )r¼   r˜   r¯   )r:   Zcoarser,   r,   r-   Úgroup_matcher  s    þþzNextViT.group_matcherTc                 C   s"   || _ | jD ]}|j|d� qd S )N)r�   )r’   rÀ   rž   )r:   r�   rÆ   r,   r,   r-   rž   "  s    
zNextViT.set_grad_checkpointingc                 C   s   | j jS r@   )rÁ   Úfcrh   r,   r,   r-   Úget_classifier(  s    zNextViT.get_classifierc                 C   s   | j j||d� d S )N)r¶   )rÁ   Úreset)r:   r·   rÃ   r,   r,   r-   Úreset_classifier,  s    zNextViT.reset_classifierc                 C   s@   |   |¡}| jr(tj ¡ s(t| j|ƒ}n
|  |¡}|  |¡}|S r@   )r¼   r’   r   r�   rŸ   r   rÀ   r8   rA   r,   r,   r-   Úforward_features/  s    


zNextViT.forward_features©Ú
pre_logitsc                 C   s   |r| j ||d�S |   |¡S )NrÒ   )rÁ   )r:   rB   rÓ   r,   r,   r-   Úforward_head8  s    zNextViT.forward_headc                 C   s   |   |¡}|  |¡}|S r@   )rÑ   rÔ   rA   r,   r,   r-   rC   ;  s    

zNextViT.forward)F)T)N)F)rD   rE   rF   r   rG   r6   rÂ   r   r�   r    rÌ   rž   rÎ   rÐ   rÑ   rƒ   rÔ   rC   rI   r,   r,   r>   r-   r¡   »  s2   ñQ	

	r¡   c                 C   sP   d| kr| S |  ¡ }i }t| ¡ |  ¡ | ¡ |  ¡ ƒD ]\}}}}|||< q6|S )z$ Remap original checkpoints -> timm zhead.fc.weight)Ú
state_dictÚzipÚkeysÚvalues)rÕ   ÚmodelÚDZout_dictÚkaÚkbÚvaZvbr,   r,   r-   Úcheckpoint_filter_fnA  s    *
rÞ   Fc                 K   sP   t dd„ t| dd¡ƒD ƒƒ}| d|¡}tt| |fttd|d�dœ|—Ž}|S )	Nc                 s   s   | ]\}}|V  qd S r@   r,   )r±   r²   Ú_r,   r,   r-   Ú	<genexpr>P  s     z"_create_nextvit.<locals>.<genexpr>r»   )r   r   r   r   Úout_indicesT)Zflatten_sequentialrá   )Zpretrained_filter_fnZfeature_cfg)r•   r“   ÚgetÚpopr   r¡   rÞ   r°   )ÚvariantÚ
pretrainedÚkwargsZdefault_out_indicesrá   rÙ   r,   r,   r-   Ú_create_nextvitO  s    ý
ûúrç   Ú c                 K   s   | dddddt tdddœ
|–S )	Nr¢   )r   éà   ré   )é   rê   gffffffî?Zbicubiczstem.0.convzhead.fc)
Úurlr·   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationZmeanrÇ   Z
first_convÚ
classifierr   )rë   ræ   r,   r,   r-   Ú_cfg^  s         ûúrñ   ztimm/)Ú	hf_hub_id)r   r­   r­   )é   ró   r‘   )rò   rì   rí   rî   )znextvit_small.bd_in1kznextvit_base.bd_in1kznextvit_large.bd_in1kznextvit_small.bd_in1k_384znextvit_base.bd_in1k_384znextvit_large.bd_in1k_384znextvit_small.bd_ssld_6m_in1kznextvit_base.bd_ssld_6m_in1kznextvit_large.bd_ssld_6m_in1kz!nextvit_small.bd_ssld_6m_in1k_384z nextvit_base.bd_ssld_6m_in1k_384z!nextvit_large.bd_ssld_6m_in1k_384c                 K   s*   t ddd�}tdd| it |f|Ž—Ž}|S )Nr¥   r©   ©r»   rÅ   Únextvit_smallrå   )rõ   ©r°   rç   ©rå   ræ   Z
model_argsrÙ   r,   r,   r-   rõ   ˜  s     ÿÿ
ÿrõ   c                 K   s*   t ddd�}tdd| it |f|Ž—Ž}|S )N)r   r¦   é   r   çš™™™™™É?rô   Únextvit_baserå   )rú   rö   r÷   r,   r,   r-   rú      s     ÿÿ
ÿrú   c                 K   s*   t ddd�}tdd| it |f|Ž—Ž}|S )N)r   r¦   é   r   rù   rô   Únextvit_largerå   )rü   rö   r÷   r,   r,   r-   rü   ¨  s     ÿÿ
ÿrü   )N)N)F)rè   )F)F)F).rX   Ú	functoolsr   r   Ztorch.nn.functionalr   Z
functionalr|   Z	timm.datar   r   Ztimm.layersr   r   r	   r
   r   r   r   Z_builderr   Z_features_fxr   Z_manipulater   Ú	_registryr   r   r.   ÚModuler/   rN   rO   rR   rY   rk   r…   r�   r¡   rÞ   rç   rñ   Zdefault_cfgsrõ   rú   rü   r,   r,   r,   r-   Ú<module>   s®    
/

:F]E 

ÿÿÿ  þ  þ  þÿÿÿ  þ  þ  þØ/