U ʹjc`@sdZddlZddlZddlmZddlmZmZmZm Z zddlm Z Wn e k rhddl m Z YnXddl Z ddlmZddlmZddlmZddlmZmZdd lmZmZmZmZmZmZmZd d lm Z d d l!m"Z"d d l#m$Z$m%Z%d dl&m'Z'dgZ(e)e*Z+Gdddej,Z-Gdddej,Z.Gdddej,Z/Gdddej,Z0Gdddej,Z1dJddZ2dKddZ3e$e3dd d!d"e3dd#d$e3d%d d&e3d'd d&e3d(d d&e3d)d d&e3d*d d&e3d+d d&e3e3d,d d&e3e3d-d d&e3d. Z4e%dLe1d/d0d1Z5e%dMe1d/d2d3Z6e%dNe1d/d4d5Z7e%dOe1d/d6d7Z8e%dPe1d/d8d9Z9e%dQe1d/d:d;Z:e%dRe1d/dd?Ze%dVe1d/dDdEZ?e%dWe1d/dFdGZ@e%dXe1d/dHdIZAdS)Yz Relative Position Vision Transformer (ViT) in PyTorch NOTE: these models are experimental / WIP, expect changes Hacked together by / Copyright 2022, Ross Wightman N)partial)OptionalTupleTypeUnion)Literal)Final) checkpointIMAGENET_INCEPTION_MEANIMAGENET_INCEPTION_STD) PatchEmbedMlpDropPath RelPosMlp RelPosBiasuse_fused_attn LayerType)build_model_with_cfg) named_apply)generate_default_cfgsregister_model)get_init_weights_vitVisionTransformerRelPoscsReZdZUeeed<ddddddejffdd Zd e e j dd d Z Z S) RelPosAttention fused_attnFNc st||dkstd||_|||_|jd|_t|_tj ||d|d|_ |rf||jnt |_ |r~||jnt |_ |r||dnd|_t||_t |||_t||_dS)Nrz$dim should be divisible by num_headsg)Zbias num_heads)super__init__AssertionErrorr!head_dimscalerrnnLinearqkvIdentityq_normk_normrel_posDropout attn_dropproj proj_drop) selfdimr!qkv_biasqk_norm rel_pos_clsr/r1 norm_layer __class__d/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/vision_transformer_relpos.pyr#%s    zRelPosAttention.__init__shared_rel_posc Cs@|j\}}}||||d|j|jddddd}|d\}}} ||}||}|j r|j dk rv|j } n|dk r|} nd} t j jj||| | |jr|jjndd}nb||j}||dd } |j dk r|j | |d } n|dk r| |} | jd d } || } | | }|dd|||}||}||}|S) Nrrrr)Z attn_maskZ dropout_pr<r3)shaper)Zreshaper!r%ZpermuteZunbindr+r,rr-get_biastorchr'Z functionalZscaled_dot_product_attentionZtrainingr/pr&Z transposeZsoftmaxr0r1) r2xr=BNCr)qkvZ attn_biasattnr:r:r;forward?s> *          zRelPosAttention.forward)N)__name__ __module__ __qualname__rbool__annotations__r' LayerNormr#rrETensorrO __classcell__r:r:r8r;r"s  rcs&eZdZdfdd ZddZZS) LayerScaleh㈵>Fcs*t||_t|t||_dSN)r"r#inplacer' ParameterrEZonesgamma)r2r3 init_valuesr[r8r:r;r#es zLayerScale.__init__cCs|jr||jS||jSrZ)r[Zmul_r]r2rGr:r:r;rOjszLayerScale.forward)rYF)rPrQrRr#rOrWr:r:r8r;rXdsrXc sLeZdZddddddddejejf fdd Zd eej ddd Z Z S) RelPosBlock@FNrc st| ||_t|||||| |d|_|rrjNrskip)r{tokenmap)rjaxmocor)img_size patch_sizein_chans num_classes global_pool embed_dimdepthr!rnr4r5r^ class_tokenfc_norm rel_pos_type rel_pos_dimr= drop_rateproj_drop_rateattn_drop_ratedrop_path_rate weight_initfix_init embed_layerr7rdblock_fncst|dkst| s&|dks&tp6ttjddp@tj||_||_|_ |_ | rbdnd|_ d|_ ||||d|_ |j j}t||j d }|d r|r||d <d |krd |d <ttf| n ttf| d|_|r d|_d | r ttd|j nd|_ddtd||Dt f ddt|D|_|snnt|_|rnt|_t||_ |dkrt!|j |nt|_"|dkr|#||r|$dS)aE Args: img_size: input image size patch_size: patch size in_chans: number of input channels num_classes: number of classes for classification head global_pool: type of global pooling for final sequence (default: 'avg') embed_dim: embedding dimension depth: depth of transformer num_heads: number of attention heads mlp_ratio: ratio of mlp hidden dim to embedding dim qkv_bias: enable bias for qkv if True qk_norm: Enable normalization of query and key in attention init_values: layer-scale init values class_token: use class token (default: False) fc_norm: use pre classifier norm instead of pre-pool rel_pos_type: type of relative position shared_rel_pos: share relative pos across all blocks drop_rate: dropout rate proj_drop_rate: projection dropout rate attn_drop_rate: attention dropout rate drop_path_rate: stochastic depth rate weight_init: weight init scheme fix_init: apply weight initialization fix (scaling w/ layer index) embed_layer: patch embedding layer norm_layer: normalization layer act_layer: MLP activation layer rr{rrr~)epsrrF)rrrr)Z window_sizeZ prefix_tokensrjZ hidden_dimZswinmodeNr cSsg|] }|qSr:)item).0rGr:r:r; >sz4VisionTransformerRelPos.__init__..cs0g|](}  |d qS)) r3r!rnr4r5r6r^r1r/ror7rdr:)ri rdrrZdprrr^rnr7r!rr5r4r6r:r;r?sr)%r"r#r$rr'rUrrrrZ num_featuresrnum_prefix_tokensgrad_checkpointing patch_embedZ grid_sizedict startswithrrr=r\rEZzeros cls_tokenZlinspaceZ ModuleListrangeblocksr*normrr. head_dropr(headrtfix_init_weight)r2rrrrrrrr!rnr4r5r^rrrrr=rrrrrrrr7rdrZ feat_sizeZ rel_pos_argsr8rr;r#sT:       "&  "  z VisionTransformerRelPos.__init__rcCs:|dks t|jdk r(tjj|jddtt||dS)N)rrrr~)std)r$rr'ruZnormal_rr)r2rr:r:r;rt[s  z$VisionTransformerRelPos.init_weightscCsLdd}t|jD]4\}}||jjjj|d||jjjj|dqdS)NcSs|td|dS)Ng@)Zdiv_mathsqrt)paramZ _layer_idr:r:r;rescalebsz8VisionTransformerRelPos.fix_init_weight..rescaler) enumeraterrNr0rvdatarjZfc2)r2rZlayer_idlayerr:r:r;rasz'VisionTransformerRelPos.fix_init_weightcCsdhS)Nrr:rwr:r:r;no_weight_decayisz'VisionTransformerRelPos.no_weight_decaycCstdddgdS)Nz^cls_token|patch_embed)z^blocks\.(\d+)N)z^norm)i)stemr)r)r2Zcoarser:r:r; group_matchermsz%VisionTransformerRelPos.group_matchercCs ||_dSrZ)r)r2enabler:r:r;set_grad_checkpointingtsz.VisionTransformerRelPos.set_grad_checkpointingcCs|jSrZ)rrwr:r:r;get_classifierxsz&VisionTransformerRelPos.get_classifier)rcCsD||_|dk r |dkst||_|dkr6t|j|nt|_dS)Nrr)rr$rr'r(rr*r)r2rrr:r:r;reset_classifier|s  z(VisionTransformerRelPos.reset_classifiercCs||}|jdk r8tj|j|jddd|fdd}|jdk rL|jnd}|jD]0}|j rztj szt |||d}qV|||d}qV| |}|S)NrrArrBr<)rrrEcatexpandrCr=rDrrjitZ is_scriptingr r)r2rGr=Zblkr:r:r;forward_featuress  $  z(VisionTransformerRelPos.forward_features) pre_logitscCsd|jr>|jdkr.|dd|jdfjddn|dddf}||}||}|rZ|S||S)Nr{rrBr)rrmeanrrr)r2rGrr:r:r; forward_heads 8  z$VisionTransformerRelPos.forward_headcCs||}||}|SrZ)rrr_r:r:r;rOs  zVisionTransformerRelPos.forward)r)F)T)N)F)!rPrQrR__doc__r r`rrirrfloatrSrstrrr'Modulerr#rtrrErignorerrrrrrrrOrWr:r:r8r;rs |     FcKs(|ddrtdtt||f|}|S)NZ features_onlyzs    $    B 07J   (