U
    ¯Ê¹jb:  ã                   @   s  d Z ddlZddlZddlmZ ddlmZmZ ddlZddlm	Z	 ddl
mZmZ ddlmZmZmZ dd	lmZ dd
lmZmZ ddlmZ dgZG dd„ de	jƒZG dd„ de	jƒZG dd„ de	jƒZG dd„ de	jƒZG dd„ de	jƒZdd„ Z d4dd„Z!d5dd„Z"ee"dd�e"dd�e"dd�e"dd�e"dd d!�e"dd d!�e"dd d!�e"dd d!�d"œƒZ#ed6ed#œd$d%„ƒZ$ed7ed#œd&d'„ƒZ%ed8ed#œd(d)„ƒZ&ed9ed#œd*d+„ƒZ'ed:ed#œd,d-„ƒZ(ed;ed#œd.d/„ƒZ)ed<ed#œd0d1„ƒZ*ed=ed#œd2d3„ƒZ+dS )>a€   Pooling-based Vision Transformer (PiT) in PyTorch

A PyTorch implement of Pooling-based Vision Transformers as described in
'Rethinking Spatial Dimensions of Vision Transformers' - https://arxiv.org/abs/2103.16302

This code was adapted from the original version at https://github.com/naver-ai/pit, original copyright below.

Modifications for timm by / Copyright 2020 Ross Wightman
é    N)Úpartial)ÚSequenceÚTuple)Únn©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Útrunc_normal_Ú	to_2tupleÚ	LayerNormé   )Úbuild_model_with_cfg)Úregister_modelÚgenerate_default_cfgs)ÚBlockÚPoolingVisionTransformerc                       sH   e Zd ZdZ‡ fdd„Zeejejf eejejf dœdd„Z‡  Z	S )ÚSequentialTuplezI This module exists to work around torchscript typing issues list -> listc                    s   t t| ƒj|Ž  d S ©N)Úsuperr   Ú__init__)ÚselfÚargs©Ú	__class__© úN/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/pit.pyr   "   s    zSequentialTuple.__init__©ÚxÚreturnc                 C   s   | D ]}||ƒ}q|S r   r   )r   r   Úmoduler   r   r   Úforward%   s    
zSequentialTuple.forward)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚtorchÚTensorr    Ú__classcell__r   r   r   r   r       s   r   c                       sF   e Zd Zd‡ fdd„	Zeejejf eejejf dœdd„Z‡  ZS )	ÚTransformerNç        c
           
         s^   t t| ƒ ¡  |ˆ ‰|| _|	r(|	ˆƒnt ¡ | _tj‡ ‡‡‡‡‡fdd„t|ƒD ƒŽ | _	d S )Nc                    s2   g | ]*}t ˆˆˆd ˆˆ ˆ| ttjdd�d�‘qS )Tç�íµ ÷Æ°>©Úeps)ÚdimZ	num_headsÚ	mlp_ratioZqkv_biasÚ	proj_dropÚ	attn_dropZ	drop_pathÚ
norm_layer)r   r   r   r   )Ú.0Úi©r0   Údrop_path_probÚ	embed_dimÚheadsr.   r/   r   r   Ú
<listcomp>=   s   öøz(Transformer.__init__.<locals>.<listcomp>)
r   r(   r   Úpoolr   ÚIdentityÚnormÚ
SequentialÚrangeÚblocks)
r   Zbase_dimÚdepthr7   r.   r9   r/   r0   r5   r1   r   r4   r   r   ,   s    õzTransformer.__init__r   c                 C   sº   |\}}|j d }| jd k	r,|  ||¡\}}|j \}}}}| d¡ dd¡}tj||fdd�}|  |¡}|  |¡}|d d …d |…f }|d d …|d …f }| dd¡ ||||¡}||fS )Nr   é   )r-   )	Úshaper9   ÚflattenZ	transposer%   Úcatr;   r>   Zreshape)r   r   Ú
cls_tokensZtoken_lengthÚBÚCÚHÚWr   r   r   r    J   s    



zTransformer.forward)Nr)   r)   NN©	r!   r"   r#   r   r   r%   r&   r    r'   r   r   r   r   r(   +   s        ör(   c                       s8   e Zd Zd‡ fdd„	Zeejejf dœdd„Z‡  ZS )ÚPoolingÚzerosc              	      sB   t t| ƒ ¡  tj|||d |d |||d�| _t ||¡| _d S )Nr   r@   )Úkernel_sizeÚpaddingÚstrideÚpadding_modeÚgroups)r   rJ   r   r   ÚConv2dÚconvÚLinearÚfc)r   Z
in_featureZout_featurerN   rO   r   r   r   r   _   s    ù	zPooling.__init__©r   c                 C   s   |   |¡}|  |¡}||fS r   )rR   rT   )r   r   Ú	cls_tokenr   r   r   r    m   s    

zPooling.forward)rK   rI   r   r   r   r   rJ   ^   s   rJ   c                       s2   e Zd Zd
eeeedœ‡ fdd„Zdd	„ Z‡  ZS )ÚConvEmbeddingéà   é   é   r   )Úimg_sizeÚ
patch_sizerN   rM   c                    s¨   t t| ƒ ¡  |}t|ƒ| _t|ƒ| _t | jd d|  | jd  | d ¡| _t | jd d|  | jd  | d ¡| _	| j| j	f| _
tj|||||dd�| _d S )Nr   r@   r   T)rL   rN   rM   Úbias)r   rW   r   r
   r[   r\   ÚmathÚfloorÚheightÚwidthZ	grid_sizer   rQ   rR   )r   Zin_channelsZout_channelsr[   r\   rN   rM   r   r   r   r   t   s    	

,,    þzConvEmbedding.__init__c                 C   s   |   |¡}|S r   )rR   ©r   r   r   r   r   r    ‰   s    
zConvEmbedding.forward)rX   rY   rZ   r   )r!   r"   r#   Úintr   r    r'   r   r   r   r   rW   s   s       ùùrW   c                       s´   e Zd ZdZd'eeeeee ee ee edœ‡ fdd„Zdd„ Z	e
jjdd„ ƒZe
jjd(dd„ƒZe
jjd)dd„ƒZdd„ Zd*dd„Zd d!„ Zd+ee
jd"œd#d$„Zd%d&„ Z‡  ZS ),r   z¤ Pooling-based Vision Transformer

    A PyTorch implement of 'Rethinking Spatial Dimensions of Vision Transformers'
        - https://arxiv.org/abs/2103.16302
    rX   rY   rZ   Úoverlap©é0   rf   rf   ©r@   é   é   ©r@   ri   rZ   ri   éè  é   ÚtokenFr)   )r[   r\   rN   Ú	stem_typeÚ	base_dimsr?   r7   r.   c                    s6  t t| ƒ ¡  |dkst‚|| _|| _|d |d  }|	| _|| _|rJdnd| _g | _	t
|
||||ƒ| _t t d|| jj| jj¡¡| _t t d| j|¡¡| _tj|d�| _g }dd„ t d|t|ƒ¡ |¡D ƒ}|}tt|ƒƒD ]Ž}d }|| ||  }|dk�rt||dd�}|t|| || || |||||| d	�g7 }|}|  j	t||d d|  d
|› �d�g7  _	qät|Ž | _tj |d |d  dd�| _!| | _"| _#t |¡| _$|	dk�rÊt %| j#|	¡nt &¡ | _'d | _(|�r|	dk�rút %| j#| j¡nt &¡ | _(d| _)t*| jdd� t*| jdd� |  +| j,¡ d S )N)rm   r   r@   r   )Úpc                 S   s   g | ]}|  ¡ ‘qS r   )Útolist)r2   r   r   r   r   r8   º   s     z5PoolingVisionTransformer.__init__.<locals>.<listcomp>)rN   )r9   r/   r0   r5   útransformers.)Znum_chsZ	reductionr   éÿÿÿÿr*   r+   Fg{®Gáz”?)Ústd)-r   r   r   ÚAssertionErrorro   r7   Únum_classesÚglobal_poolZ
num_tokensZfeature_inforW   Úpatch_embedr   Ú	Parameterr%   Zrandnr`   ra   Ú	pos_embedrV   ZDropoutÚpos_dropZlinspaceÚsumÚsplitr=   ÚlenrJ   r(   Údictr   Útransformersr   r;   Znum_featuresr6   Ú	head_droprS   r:   ÚheadÚ	head_distÚdistilled_trainingr	   ÚapplyÚ_init_weights)r   r[   r\   rN   rn   ro   r?   r7   r.   rv   Zin_chansrw   Ú	distilledZ	drop_rateZpos_drop_drateZproj_drop_rateZattn_drop_rateZdrop_path_rater6   r€   ZdprZprev_dimr3   r9   r   r   r   r   ”   sb     "
ýø

.
"$z!PoolingVisionTransformer.__init__c                 C   s0   t |tjƒr,tj |jd¡ tj |jd¡ d S )Nr   g      ð?)Ú
isinstancer   r   ÚinitZ	constant_r]   Úweight)r   Úmr   r   r   r†   â   s    z&PoolingVisionTransformer._init_weightsc                 C   s   ddhS )Nrz   rV   r   ©r   r   r   r   Úno_weight_decayç   s    z(PoolingVisionTransformer.no_weight_decayTc                 C   s
   || _ d S r   )r„   ©r   Úenabler   r   r   Úset_distilled_trainingë   s    z/PoolingVisionTransformer.set_distilled_trainingc                 C   s   |rt dƒ‚d S )Nz$gradient checkpointing not supported)ru   rŽ   r   r   r   Úset_grad_checkpointingï   s    z/PoolingVisionTransformer.set_grad_checkpointingc                 C   s    | j d k	r| j| j fS | jS d S r   )rƒ   r‚   rŒ   r   r   r   Úget_classifieró   s    
z'PoolingVisionTransformer.get_classifierNc                 C   sV   || _ |dkrt | j|¡nt ¡ | _| jd k	rR|dkrHt | j| j ¡nt ¡ | _d S )Nr   )rv   r   rS   r6   r:   r‚   rƒ   )r   rv   rw   r   r   r   Úreset_classifierù   s     
z)PoolingVisionTransformer.reset_classifierc                 C   sP   |   |¡}|  || j ¡}| j |jd dd¡}|  ||f¡\}}|  |¡}|S )Nr   rs   )rx   r{   rz   rV   ÚexpandrA   r€   r;   )r   r   rD   r   r   r   Úforward_featuresÿ   s    

z)PoolingVisionTransformer.forward_features)Ú
pre_logitsr   c                 C   sÌ   | j d k	r’| jdkst‚|d d …df |d d …df  }}|  |¡}|  |¡}|sf|  |¡}|   |¡}| jr„| jr„tj 	¡ s„||fS || d S n6| jdkr¬|d d …df }|  |¡}|sÄ|  |¡}|S d S )Nrm   r   r   r@   )
rƒ   rw   ru   r�   r‚   r„   Ztrainingr%   ÚjitZis_scripting)r   r   r–   Zx_distr   r   r   Úforward_head  s"    
"






z%PoolingVisionTransformer.forward_headc                 C   s   |   |¡}|  |¡}|S r   )r•   r˜   rb   r   r   r   r      s    

z PoolingVisionTransformer.forward)rX   rY   rZ   rd   re   rg   rj   ri   rk   rl   rm   Fr)   r)   r)   r)   r)   )T)T)N)F)r!   r"   r#   r$   rc   Ústrr   Úfloatr   r†   r%   r—   Úignorer�   r�   r‘   r’   r“   r•   Úboolr&   r˜   r    r'   r   r   r   r   r   Ž   sP                    î÷N

c                 C   s<   i }t  d¡}|  ¡ D ] \}}| dd„ |¡}|||< q|S )z preprocess checkpoints zpools\.(\d)\.c                 S   s   dt |  d¡ƒd › d�S )Nrr   r   z.pool.)rc   Úgroup)Úexpr   r   r   Ú<lambda>-  ó    z&checkpoint_filter_fn.<locals>.<lambda>)ÚreÚcompileÚitemsÚsub)Z
state_dictÚmodelZout_dictZp_blocksÚkÚvr   r   r   Úcheckpoint_filter_fn$  s    

r¨   Fc                 K   s@   t tdƒƒ}| d|¡}tt| |fttdd|d�dœ|—Ž}|S )Nrl   Úout_indicesÚhookT)Zfeature_clsZ
no_rewriter©   )Zpretrained_filter_fnZfeature_cfg)Útupler=   Úpopr   r   r¨   r   )ÚvariantÚ
pretrainedÚkwargsZdefault_out_indicesr©   r¥   r   r   r   Ú_create_pit2  s    ýûúr°   Ú c                 K   s    | ddd dddt tdddœ|–S )	Nrk   )rl   rX   rX   gÍÌÌÌÌÌì?ZbicubicTzpatch_embed.convr‚   )Úurlrv   Z
input_sizeZ	pool_sizeZcrop_pctÚinterpolationZfixed_input_sizeZmeanrt   Z
first_convÚ
classifierr   )r²   r¯   r   r   r   Ú_cfgA  s          ûúrµ   ztimm/)Ú	hf_hub_id)r‚   rƒ   )r¶   r´   )zpit_ti_224.in1kzpit_xs_224.in1kzpit_s_224.in1kzpit_b_224.in1kzpit_ti_distilled_224.in1kzpit_xs_distilled_224.in1kzpit_s_distilled_224.in1kzpit_b_distilled_224.in1krU   c                 K   s<   t dddddgdddgdddgdd	�}td
| ft |f|ŽŽS )Né   é   é@   rl   rh   ri   rZ   rY   ©r\   rN   ro   r?   r7   r.   Ú	pit_b_224©r   r°   ©r®   r¯   Z
model_argsr   r   r   r»   a  s    úr»   c                 K   s<   t dddddgdddgdddgdd	�}td
| ft |f|ŽŽS )NrY   rZ   rf   r@   rh   ri   rl   é   rº   Ú	pit_s_224r¼   r½   r   r   r   r¿   n  s    úr¿   c                 K   s<   t dddddgdddgdddgdd�}td| ft |f|ŽŽS )	NrY   rZ   rf   r@   rh   ri   rº   Ú
pit_xs_224r¼   r½   r   r   r   rÀ   {  s    úrÀ   c                 K   s<   t dddddgdddgdddgdd�}td| ft |f|ŽŽS )	NrY   rZ   é    r@   rh   ri   rº   Ú
pit_ti_224r¼   r½   r   r   r   rÂ   ˆ  s    úrÂ   c              	   K   s>   t dddddgdddgdddgdd	d
�}td| ft |f|ŽŽS )Nr·   r¸   r¹   rl   rh   ri   rZ   rY   T©r\   rN   ro   r?   r7   r.   r‡   Úpit_b_distilled_224r¼   r½   r   r   r   rÄ   •  s    ù	rÄ   c              	   K   s>   t dddddgdddgdddgdd	d
�}td| ft |f|ŽŽS )NrY   rZ   rf   r@   rh   ri   rl   r¾   TrÃ   Úpit_s_distilled_224r¼   r½   r   r   r   rÅ   £  s    ù	rÅ   c              	   K   s>   t dddddgdddgdddgddd�}td	| ft |f|ŽŽS )
NrY   rZ   rf   r@   rh   ri   TrÃ   Úpit_xs_distilled_224r¼   r½   r   r   r   rÆ   ±  s    ù	rÆ   c              	   K   s>   t dddddgdddgdddgddd�}td	| ft |f|ŽŽS )
NrY   rZ   rÁ   r@   rh   ri   TrÃ   Úpit_ti_distilled_224r¼   r½   r   r   r   rÇ   ¿  s    ù	rÇ   )F)r±   )F)F)F)F)F)F)F)F),r$   r^   r¡   Ú	functoolsr   Útypingr   r   r%   r   Z	timm.datar   r   Ztimm.layersr	   r
   r   Z_builderr   Ú	_registryr   r   Zvision_transformerr   Ú__all__r<   r   ÚModuler(   rJ   rW   r   r¨   r°   rµ   Zdefault_cfgsr»   r¿   rÀ   rÂ   rÄ   rÅ   rÆ   rÇ   r   r   r   r   Ú<module>   sv   3 

þþþþñ