U ʹj\@sXdZdgZddlZddlZddlmZddlmZddlZddl m Z ddl m m Z ddlmZmZddlmZmZmZmZmZmZdd lmZdd lmZdd lmZdd lmZm Z Gd ddej j!Z"Gddde j#Z$Gddde j#Z%Gddde j#Z&Gddde j#Z'Gddde j#Z(Gdddej j#Z)Gddde j#Z*ee*Gddde j#Z+Gddde j#Z,d d!Z-d?d#d$Z.e e.d%d&d'e.d%d(e.d%d(e.d%d&d'e.d%d(e.d%d(e.d%d&d'e.d%d(e.d%d(e.d%d)d*d+d,e.d%d-d.d+d/d0d1 Z/d@d3d4Z0edAd5d6Z1edBd7d8Z2edCd9d:Z3edDd;d<Z4edEd=d>Z5dS)Fz TinyViT Paper: `TinyViT: Fast Pretraining Distillation for Small Vision Transformers` - https://arxiv.org/abs/2207.10666 Adapted from official impl at https://github.com/microsoft/Cream/tree/main/TinyViT TinyVitN)partial)DictIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD) LayerNorm2dNormMlpClassifierHeadDropPath trunc_normal_resize_rel_pos_bias_table_levituse_fused_attn)build_model_with_cfg)register_notrace_module)checkpoint_seq)register_modelgenerate_default_cfgscs.eZdZdfdd ZeddZZS)ConvNormrrc s^ttj|||||||dd|_t||_tjj |jj |tjj |jj ddS)NF)biasr) super__init__nnConv2dconvZ BatchNorm2dbntorchinit constant_weightr) selfin_chsout_chsksstridepaddilationgroupsbn_weight_init __class__S/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/tiny_vit.pyrs   zConvNorm.__init__c Cs|j|j}}|j|j|jd}|j|dddddf}|j|j|j|j|jd}tjj | d|jj | d|j dd|jj |jj|jj|jj d}|jj||jj||S)Ng?rr)r$paddingr&r')rrrZ running_varepsrZ running_meanrrrsizer'shaper$r.r&dataZcopy_)r crwbmr+r+r,fuse%s$ z ConvNorm.fuse)rrrrrr)__name__ __module__ __qualname__rrno_gradr7 __classcell__r+r+r)r,rsrcs$eZdZfddZddZZS) PatchEmbedcsHtd|_t||dddd|_||_t|d|ddd|_dS)Nr-r)rrr$rconv1actconv2)r r!r" act_layerr)r+r,r5s  zPatchEmbed.__init__cCs"||}||}||}|SN)r@rArBr xr+r+r,forward<s   zPatchEmbed.forwardr8r9r:rrGr<r+r+r)r,r=4s r=cs$eZdZfddZddZZS)MBConvcstt||}t||dd|_||_t||ddd|d|_||_t||ddd|_||_ |dkrvt |nt |_ dS)Nr)r#r?r#r$r%r')r#r()rrintrr@act1rBact2conv3act3r rIdentity drop_path)r r!r"Z expand_ratiorCrRZmid_chsr)r+r,rDs  zMBConv.__init__cCsV|}||}||}||}||}||}||}||7}||}|SrD)r@rMrBrNrOrRrP)r rFshortcutr+r+r,rGOs       zMBConv.forwardrHr+r+r)r,rICs rIcs$eZdZfddZddZZS) PatchMergingcsXtt||ddd|_||_t||ddd|d|_||_t||ddd|_dS)Nrrr?r-)r')rrrr@rMrBrNrO)r dimout_dimrCr)r+r,r]s  zPatchMerging.__init__cCs6||}||}||}||}||}|SrD)r@rMrBrNrOrEr+r+r,rGes      zPatchMerging.forwardrHr+r+r)r,rT\s rTcs&eZdZdfdd ZddZZS) ConvLayerrK@cs>t|_||_tjfddt|D|_dS)Nc s.g|]&}tttr$|nqSr+)rI isinstancelist.0irCconv_expand_ratiorUrRr+r, zsz&ConvLayer.__init__..)rrrUdepthr Sequentialrangeblocks)r rUrarCrRr_r)r^r,ros  zConvLayer.__init__cCs||}|SrD)rdrEr+r+r,rGs zConvLayer.forward)rKrXrHr+r+r)r,rWnsrWcs4eZdZddejejdffdd ZddZZS)NormMlpNrKcsdt|p|}|p|}|||_t|||_||_t||_t|||_ t||_ dSrD) rrnormrLinearfc1rAZDropoutdrop1fc2drop2)r in_featureshidden_featuresZ out_features norm_layerrCdropr)r+r,rs   zNormMlp.__init__cCs@||}||}||}||}||}||}|SrD)rfrhrArirjrkrEr+r+r,rGs      zNormMlp.forward) r8r9r:r LayerNormGELUrrGr<r+r+r)r,resrecsteZdZUejjeed<ee ej fed<dfdd Z e dfd d Z ejej d d d ZddZZS) Attention fused_attnattention_bias_cacher>rwc srtt|tr t|dks$t||_|d|_||_t |||_ |j ||_ ||_ ||_ t|_t||_t|||j d||_t|j ||_ttt|dt|d}t|}i}g} |D]Z} |D]P} t| d| dt| d| df} | |krt||| <| || qqtjt|t||_|j dt!| "||ddi|_#dS)Nr-grrattention_bias_idxsF) persistent)$rrrYtuplelenAssertionError num_headsscalekey_dimrLval_dimrV attn_ratio resolutionr rsrrprfrgqkvprojrZ itertoolsproductrcabsappendr ParameterZzerosattention_biasesZregister_bufferZ LongTensorviewrt) r rUrr}rrZpointsNZattention_offsetsZidxsp1Zp2offsetr)r+r,rs4     (  zAttention.__init__Tcs t||r|jri|_dSrD)rtrainrt)r moder)r+r,rs  zAttention.train)devicereturncCs^tjs|jr$|jdd|jfSt|}||jkrP|jdd|jf|j|<|j|SdSrD)rjit is_tracingZtrainingrrxstrrt)r rZ device_keyr+r+r,get_attention_biasess  zAttention.get_attention_biasesc Cs||j}|j\}}}||}||}||||jdj|j|j|j gdd\}}} | dddd}| dddd}| dddd} |j rt j ||| |d}n6||j}||dd} | |} | jdd} | | }|dd|||j}||}|S) Nr?)rUrr-r)Z attn_mask)rrr1rfrrr}splitrrpermutersFZscaled_dot_product_attentionr~ transposeZsoftmaxreshaperVr) r rFZ attn_biasBr_rqkvattnr+r+r,rGs$    .   zAttention.forward)rur>rv)T)r8r9r:rrFinalbool__annotations__rrZTensorrr;rrrrGr<r+r+r)r,rrs % rrcsFeZdZdZdddddejffdd Zdd Zed d d Z Z S) TinyVitBlocka5 TinyViT Block. Args: dim (int): Number of input channels. num_heads (int): Number of attention heads. window_size (int): Window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. drop (float, optional): Dropout rate. Default: 0.0 drop_path (float, optional): Stochastic depth rate. Default: 0.0 local_conv_size (int): the kernel size of the convolution between Attention and MLP. Default: 3 act_layer: the activation function. Default: nn.GELU rXrKr?c st||_||_|dks&td||_||_||dksFtd||} ||f} t|| |d| d|_|dkrzt |nt |_ t |t||||d|_|dkrt |nt |_|d} t|||d| |d |_dS) Nrz"window_size must be greater than 0z"dim must be divisible by num_headsr)rrrK)rlrmrCror-rJ)rrrUr}r| window_size mlp_ratiorrrr rrQ drop_path1rerLmlp drop_path2r local_conv) r rUr}rrrorRlocal_conv_sizerCZhead_dimZwindow_resolutionr%r)r+r,rs(  zTinyVitBlock.__init__c Cs|j\}}}}||}|}||jkrZ||jkrZ||||}||}|||||}n|j||j|j}|j||j|j} |dkp| dk} | rt|ddd| d|f}|||| } } | |j} | |j}||| |j||j|dd|| ||j|j|}||}||| ||j|j|dd|| | |}| rl|ddd|d|f}|| |}| dddd}| |}||||dd}|| | |}|||||SNrr-r?r)r1rrrrrr%r contiguousrrrrr)r rFrHWCLrSZpad_bZpad_rr.ZpHZpWZnHZnWr+r+r,rG+s<      , zTinyVitBlock.forwardrcCs$d|jd|jd|jd|jS)Ndim=z , num_heads=z, window_size=z , mlp_ratio=)rUr}rrr r+r+r, extra_reprSszTinyVitBlock.extra_repr r8r9r:__doc__rrqrrGrrr<r+r+r)r,rs%(rcsFeZdZdZdddddejffdd Zdd Zed d d Z Z S) TinyVitStagea A basic TinyViT layer for one stage. Args: dim (int): Number of input channels. out_dim: the output dimension of the layer depth (int): Number of blocks. num_heads (int): Number of attention heads. window_size (int): Local window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. drop (float, optional): Dropout rate. Default: 0.0 drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None local_conv_size: the kernel size of the depthwise convolution between attention and MLP. Default: 3 act_layer: the activation function. Default: nn.GELU rXrKNr?c svt||_|_| dk r0| |d|_nt|_|ksFttjfddt |D|_ dS)N)rUrVrCc s6g|].}tttr&|ndqS))rUr}rrrorRrrC)rrYrZr[rCrorRrrr}rVrr+r,r`s z)TinyVitStage.__init__..) rrrarV downsamplerrQr|rbrcrd) r rUrVrar}rrrorRrrrCr)rr,rls     zTinyVitStage.__init__cCs8||}|dddd}||}|dddd}|Sr)rrrdrEr+r+r,rGs   zTinyVitStage.forwardrcCsd|jd|jS)Nrz, depth=)rVrarr+r+r,rszTinyVitStage.extra_reprrr+r+r)r,r[s,rcseZdZddddddddd d d ddejffd d ZddZejj ddZ ejj ddZ ejj d$ddZ ejj d%ddZ ejj ddZd&ddZddZd d!Zd"d#ZZS)'rr?avg)`i)r-r-r-)r?r )rrrwrrXrK皙?Fcst||_||_t||_||_| |_t||d|d|_ ddt d| t |D}t |_|j j}|d}g|_t|jD]}|dkrt|||||d||| d}np||}|t |d|t |d|d} t||d||||||||j| | | t|d }|}|d9}|j||jt||d |d g7_q|d |_ttd d }t|j|||d|_||jdS)Nr)r!r"rCcSsg|] }|qSr+)itemr\rFr+r+r,r`sz$TinyVit.__init__..)rUrarCrRr_r) rUrVrar}rrrorrRrrCr-zstages.)Znum_chsZ reductionmodulergh㈵>)r/) pool_typern)rr num_classesdepthsr{Z num_stagesrgrad_checkpointingr= patch_embedrZlinspacesumrrbstagesr$Z feature_inforcrWrrTrdictZ num_featuresrrr headapply _init_weights)r Zin_chansr global_pool embed_dimsrr} window_sizesrZ drop_ratedrop_path_rateZuse_checkpointZmbconv_expand_ratiorrCZdprr$Zprev_dimZ stage_idxZstagerVZ norm_layer_cfr)r+r,rsj   (   "  zTinyVit.__init__cCsDt|tjr@t|jddt|tjr@|jdk r@tj|jddS)Ng{Gz?)stdr)rYrrgr rrrr)r r6r+r+r,rs zTinyVit._init_weightscCsdhS)Nrr+rr+r+r,no_weight_decay_keywordssz TinyVit.no_weight_decay_keywordscCsdd|DS)NcSsh|]}d|kr|qS)rr+rr+r+r, sz*TinyVit.no_weight_decay..) state_dictkeysrr+r+r,no_weight_decayszTinyVit.no_weight_decaycCstd|r dnddgd}|S)Nz ^patch_embedz^stages\.(\d+))z^stages\.(\d+).downsample)r)z^stages\.(\d+)\.\w+\.(\d+)N)stemrd)r)r ZcoarseZmatcherr+r+r, group_matcherszTinyVit.group_matcherTcCs ||_dSrD)r)r enabler+r+r,set_grad_checkpointingszTinyVit.set_grad_checkpointingcCs|jjSrD)rfcrr+r+r,get_classifierszTinyVit.get_classifierNcCs||_|jj||ddS)N)r)rrreset)r rrr+r+r,reset_classifierszTinyVit.reset_classifiercCs6||}|jr(tjs(t|j|}n ||}|SrD)rrrrZ is_scriptingrrrEr+r+r,forward_featuress   zTinyVit.forward_featurescCs||}|SrD)rrEr+r+r, forward_head$s zTinyVit.forward_headcCs||}||}|SrD)rrrEr+r+r,rG(s  zTinyVit.forward)F)T)N)r8r9r:rrqrrrrignorerrrrrrrrrGr<r+r+r)r,rs<S      cCspd|kr|d}|}i}|D]B\}}|dr rrwrKrrr}rrtiny_vit_5m_224rupdaterrrZ model_kwargsr+r+r,rs     rcKsJtddddgddddgdddd gd d d d gd d }||td |f|S)Nrrir-rr>rurwrrrtiny_vit_11m_224rr r+r+r,r s     r cKsJtddddgddddgdddd gd d d d gd d }||td|f|S)Nrrr@r-rr?rrrwg?rtiny_vit_21m_224rr r+r+r,rs     rcKsJtddddgddddgdddd gddd dgd d }||td |f|S)Nrrrr r-rr?rrrrrtiny_vit_21m_384rr r+r+r,rs     rcKsJtddddgddddgdddd gd d d d gd d }||td|f|S)Nrrrr r-rr?rrr rrtiny_vit_21m_512rr r+r+r,rs     r)r)F)F)F)F)F)F)6r__all__mathr functoolsrtypingrrZtorch.nnrZtorch.nn.functionalZ functionalrZ timm.datarrZ timm.layersrr r r r r Z_builderrZ _features_fxrZ _manipulater _registryrrrbrModuler=rIrTrWrerrrrrrrZ default_cfgsrrr rrrr+r+r+r,s       RaH  5