U
    ¯Ê¹jòC  ã                   @   sþ  d Z ddlmZ ddlmZmZmZ ddlZddlm	Z	 ddl
m	  mZ ddlmZmZ ddlmZmZmZmZmZ ddlmZmZmZ dd	lmZmZ dd
lmZmZ ddl m!Z!m"Z" G dd„ de	j#ƒZ$G dd„ de	j#ƒZ%dVdd„Z&dWdd„Z'dXdd„Z(ee(ddddd�e(ddddd dd!�e(d"ddd#�e(d$ddd dd%�e(ƒ e(d&ddd d'�e(d(ddd#�e(d)ddd dd%�e(d*dd+d,ddd-�e(d.dd+d,dd/�e(ddd,d0�e(d1dd+d,dd/�e(eed2d3�e(eed2d3�e(eed2d3�e(eed2d3�d4œƒZ)edYe"d5œd6d7„ƒZ*edZe"d5œd8d9„ƒZ+ed[e"d5œd:d;„ƒZ,ed\e"d5œd<d=„ƒZ-ed]e"d5œd>d?„ƒZ.ed^e"d5œd@dA„ƒZ/ed_e"d5œdBdC„ƒZ0ed`e"d5œdDdE„ƒZ1edae"d5œdFdG„ƒZ2edbe"d5œdHdI„ƒZ3edce"d5œdJdK„ƒZ4edde"d5œdLdM„ƒZ5edee"d5œdNdO„ƒZ6ee7dPdQdRdRdSdTdUœƒ dS )fa   Hybrid Vision Transformer (ViT) in PyTorch

A PyTorch implement of the Hybrid Vision Transformers as described in:

'An Image Is Worth 16 x 16 Words: Transformers for Image Recognition at Scale'
    - https://arxiv.org/abs/2010.11929

`How to train your ViT? Data, Augmentation, and Regularization in Vision Transformers`
    - https://arxiv.org/abs/2106.10270

NOTE These hybrid model definitions depend on code in vision_transformer.py.
They were moved here to keep file sizes sane.

Hacked together by / Copyright 2020, Ross Wightman
é    )Úpartial)ÚListÚOptionalÚTupleN)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚStdConv2dSameÚ	StdConv2dÚ	to_2tupleÚFormatÚnchw_toé   )Úgenerate_default_cfgsÚregister_modelÚregister_model_deprecations)Ú	resnet26dÚ	resnet50d)ÚResNetV2Úcreate_resnetv2_stem)Ú_create_vision_transformerÚVisionTransformerc                
       sT   e Zd ZU dZeed< ejje	 ed< de	e
e e	e	dœ‡ fdd„Zdd„ Z‡  ZS )ÚHybridEmbedúd CNN Feature Map Embedding
    Extract feature map from CNN, flatten, project to embedding dim.
    Ú
output_fmtÚdynamic_img_padéà   r   Né   é   TF)Úflattenr   Ústrict_img_sizer   c              
      s®  t ƒ  ¡  t|tjƒst‚t|ƒ}t|ƒ}|| _|| _|| _	|d krÂt
 ¡ �n |j}|r`| ¡  |  	t
 d||d |d ¡¡}t|ttfƒr”|d }|jdd … }|jd }| |¡ W 5 Q R X n.t|ƒ}t| j	dƒrè| j	j ¡ d }n| j	j}|�s&|d |d  dk�r"|d |d  dk�s&t‚|d |d  |d |d  f| _| jd | jd  | _|	d k	�rzd| _t|	ƒ| _n|| _tj| _|
| _|| _tj|||||d�| _ d S )Nr   r   éÿÿÿÿéþÿÿÿÚfeature_infoF)Zkernel_sizeZstrideÚbias)!ÚsuperÚ__init__Ú
isinstanceÚnnÚModuleÚAssertionErrorr
   Úimg_sizeÚ
patch_sizeÚbackboneÚtorchZno_gradÚtrainingÚevalZzerosÚlistÚtupleÚshapeÚtrainÚhasattrr"   ZchannelsZnum_featuresZ	grid_sizeZnum_patchesr   r   r   ÚNCHWr   r   ZConv2dÚproj)Úselfr,   r*   r+   Úfeature_sizeÚin_chansÚ	embed_dimr#   r   r   r   r   r.   ÚoZfeature_dim©Ú	__class__© úd/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/vision_transformer_hybrid.pyr%   &   sD    


0"
zHybridEmbed.__init__c                 C   sÌ   |   |¡}t|ttfƒr |d }|j\}}}}| jrŒ| jd || jd   | jd  }| jd || jd   | jd  }t |d|d|f¡}|  	|¡}| j
r°| 
d¡ dd¡}n| jtjkrÈt|| jƒ}|S )Nr    r   r   é   )r,   r&   r0   r1   r2   r   r+   ÚFÚpadr6   r   Ú	transposer   r   r5   r   )r7   ÚxÚ_ÚHÚWZpad_hZpad_wr>   r>   r?   Úforward]   s    
""
zHybridEmbed.forward)
r   r   Nr   r   TTNTF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ú__annotations__r-   ZjitÚFinalÚboolr   Ústrr%   rH   Ú__classcell__r>   r>   r<   r?   r      s(   
          ô	ô7r   c                       s>   e Zd ZdZd‡ fdd	„	Zeejee	 f d
œdd„Z
‡  ZS )ÚHybridEmbedWithSizer   r   r   Nr   r   Tc              	      s   t ƒ j|||||||d� d S )N)r,   r*   r+   r8   r9   r:   r#   )r$   r%   )r7   r,   r*   r+   r8   r9   r:   r#   r<   r>   r?   r%   r   s    
ùzHybridEmbedWithSize.__init__©Úreturnc                 C   sJ   |   |¡}t|ttfƒr |d }|  |¡}| d¡ dd¡|jdd … fS )Nr    r@   r   r!   )r,   r&   r0   r1   r6   r   rC   r2   )r7   rD   r>   r>   r?   rH   †   s
    

zHybridEmbedWithSize.forward)r   r   Nr   r   T)rI   rJ   rK   rL   r%   r   r-   ZTensorr   ÚintrH   rQ   r>   r>   r<   r?   rR   n   s         ørR   Fc                 K   s.   t t|d�}| dd¡ t| f||dœ|—ŽS )N)r,   r+   r   )Ú
pretrainedÚembed_layer)r   r   Ú
setdefaultr   )Úvariantr,   rV   ÚkwargsrW   r>   r>   r?   Ú!_create_vision_transformer_hybridŽ   s    r[   ©r   é   é	   c              	   K   sx   |  dd¡}|rdnd}|r(ttdd�n
ttdd�}t| ƒr\t| dd|  dd	¡d
||d�}nt|  dd	¡|d
|d�}|S )z ResNet-V2 backbone helperÚpadding_sameTZsameÚ g:Œ0âŽyE>)Úepsr   r9   r   F)ÚlayersÚnum_classesZglobal_poolr9   ÚpreactÚ	stem_typeÚ
conv_layer)re   rd   rf   )Úgetr   r   r	   Úlenr   r   )rb   rZ   r_   re   rf   r,   r>   r>   r?   Ú	_resnetv2”   s(       
  þ
   ÿri   r`   c                 K   s    | ddd dddddddd	œ|–S )
Niè  )r   r   r   çÍÌÌÌÌÌì?ZbicubicT)ç      à?rk   rk   zpatch_embed.backbone.stem.convÚhead)Úurlrc   Ú
input_sizeZ	pool_sizeÚcrop_pctÚinterpolationZfixed_input_sizeÚmeanÚstdÚ
first_convÚ
classifierr>   )rm   rZ   r>   r>   r?   Ú_cfg£   s          ûúru   z—https://storage.googleapis.com/vit_models/augreg/R_Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzztimm/Tzpatch_embed.backbone.conv)rm   Ú	hf_hub_idÚcustom_loadrs   z—https://storage.googleapis.com/vit_models/augreg/R_Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz)r   é€  rx   g      ð?)rm   rv   rs   rn   ro   rw   zšhttps://storage.googleapis.com/vit_models/augreg/R26_S_32-i21k-300ep-lr_0.001-aug_light0-wd_0.03-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.03-res_224.npz)rm   rv   rw   z›https://storage.googleapis.com/vit_models/augreg/R26_S_32-i21k-300ep-lr_0.001-aug_medium2-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz)rm   rv   rn   ro   rw   zthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_base_resnet50_384-9fd3c705.pth)rm   rv   rn   ro   zšhttps://storage.googleapis.com/vit_models/augreg/R50_L_32-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzzšhttps://storage.googleapis.com/vit_models/augreg/R50_L_32-i21k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_384.npzzohttps://storage.googleapis.com/vit_models/augreg/R_Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0.npziSU  rj   )rm   rv   rc   ro   rs   rw   zshttps://storage.googleapis.com/vit_models/augreg/R26_S_32-i21k-300ep-lr_0.001-aug_medium2-wd_0.03-do_0.0-sd_0.0.npz)rm   rv   rc   ro   rw   )rv   rc   ro   zrhttps://storage.googleapis.com/vit_models/augreg/R50_L_32-i21k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0.npzzpatch_embed.backbone.conv1.0)rq   rr   rs   )z*vit_tiny_r_s16_p8_224.augreg_in21k_ft_in1kz*vit_tiny_r_s16_p8_384.augreg_in21k_ft_in1kz*vit_small_r26_s32_224.augreg_in21k_ft_in1kz*vit_small_r26_s32_384.augreg_in21k_ft_in1kzvit_base_r26_s32_224.untrainedú'vit_base_r50_s16_384.orig_in21k_ft_in1kz*vit_large_r50_s32_224.augreg_in21k_ft_in1kz*vit_large_r50_s32_384.augreg_in21k_ft_in1kú"vit_tiny_r_s16_p8_224.augreg_in21kú"vit_small_r26_s32_224.augreg_in21kúvit_base_r50_s16_224.orig_in21kú"vit_large_r50_s32_224.augreg_in21kz!vit_small_resnet26d_224.untrainedz%vit_small_resnet50d_s16_224.untrainedz vit_base_resnet26d_224.untrainedz vit_base_resnet50d_224.untrainedrS   c                 K   sB   t f ddi|—Ž}tddddd�}td
|| d	œt|f|Ž—Ž}|S )z3 R+ViT-Ti/S16 w/ 8x8 patch hybrid @ 224 x 224.
    rb   r>   é   éÀ   é   r   ©r+   r:   ÚdepthÚ	num_headsÚvit_tiny_r_s16_p8_224©r,   rV   )r„   ©ri   Údictr[   ©rV   rZ   r,   Z
model_argsÚmodelr>   r>   r?   r„   ð   s     ÿ ÿ
ÿr„   c                 K   sB   t f ddi|—Ž}tddddd�}td
|| d	œt|f|Ž—Ž}|S )z3 R+ViT-Ti/S16 w/ 8x8 patch hybrid @ 384 x 384.
    rb   r>   r~   r   r€   r   r�   Úvit_tiny_r_s16_p8_384r…   )rŠ   r†   rˆ   r>   r>   r?   rŠ   û   s     ÿ ÿ
ÿrŠ   c                 K   s8   t d|Ž}tdddd�}td	|| dœt|f|Ž—Ž}|S )
ú R26+ViT-S/S32 hybrid.
    ©r@   r@   r@   r@   rx   r€   é   ©r:   r‚   rƒ   Úvit_small_r26_s32_224r…   )rŒ   )r�   r†   rˆ   r>   r>   r?   r�     s    
 ÿ ÿ
ÿr�   c                 K   s8   t d|Ž}tdddd�}td	|| dœt|f|Ž—Ž}|S )
r‹   rŒ   rx   r€   r�   rŽ   Úvit_small_r26_s32_384r…   )rŒ   )r�   r†   rˆ   r>   r>   r?   r�     s    
 ÿ ÿ
ÿr�   c                 K   s8   t d|Ž}tdddd�}td|| dœt|f|Ž—Ž}|S )	z R26+ViT-B/S32 hybrid.
    rŒ   r   r€   rŽ   Úvit_base_r26_s32_224r…   )rŒ   )r‘   r†   rˆ   r>   r>   r?   r‘     s    
 ÿ ÿ
ÿr‘   c                 K   s8   t d|Ž}tdddd�}td|| dœt|f|Ž—Ž}|S )	zR R50+ViT-B/S16 hybrid from original paper (https://arxiv.org/abs/2010.11929).
    r\   r   r€   rŽ   Úvit_base_r50_s16_224r…   )r\   )r’   r†   rˆ   r>   r>   r?   r’   '  s    
 ÿ ÿ
ÿr’   c                 K   s8   t d|Ž}tdddd�}td|| dœt|f|Ž—Ž}|S )	zÈ R50+ViT-B/16 hybrid from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    r\   r   r€   rŽ   Úvit_base_r50_s16_384r…   )r\   )r“   r†   rˆ   r>   r>   r?   r“   2  s    
 ÿ ÿ
ÿr“   c                 K   s8   t d|Ž}tdddd�}td	|| dœt|f|Ž—Ž}|S )
ú R50+ViT-L/S32 hybrid.
    ©r   r]   r�   r   é   é   é   rŽ   Úvit_large_r50_s32_224r…   )r•   )r™   r†   rˆ   r>   r>   r?   r™   >  s    
 ÿ ÿ
ÿr™   c                 K   s8   t d|Ž}tdddd�}td	|| dœt|f|Ž—Ž}|S )
r”   r•   r–   r—   r˜   rŽ   Úvit_large_r50_s32_384r…   )r•   )rš   r†   rˆ   r>   r>   r?   rš   I  s    
 ÿ ÿ
ÿrš   c                 K   sJ   t | | dd¡ddgd�}tddddd�}td|| d
œt|f|Ž—Ž}|S )zL Custom ViT small hybrid w/ ResNet26D stride 32. No pretrained weights.
    r9   r   Tr]   ©rV   r9   Zfeatures_onlyZout_indicesr   r~   ©r:   r‚   rƒ   Z	mlp_ratioÚvit_small_resnet26d_224r…   )r�   ©r   rg   r‡   r[   rˆ   r>   r>   r?   r�   T  s     ÿ ÿ
ÿr�   c                 K   sJ   t | | dd¡ddgd�}tddddd�}td
|| d	œt|f|Ž—Ž}|S )zV Custom ViT small hybrid w/ ResNet50D 3-stages, stride 16. No pretrained weights.
    r9   r   Tr›   r   r~   rœ   Úvit_small_resnet50d_s16_224r…   )rŸ   ©r   rg   r‡   r[   rˆ   r>   r>   r?   rŸ   _  s     ÿ ÿ
ÿrŸ   c                 K   sH   t | | dd¡ddgd�}tdddd�}td|| d
œt|f|Ž—Ž}|S )zK Custom ViT base hybrid w/ ResNet26D stride 32. No pretrained weights.
    r9   r   Tr]   r›   r   r€   rŽ   Úvit_base_resnet26d_224r…   )r¡   rž   rˆ   r>   r>   r?   r¡   j  s     ÿ ÿ
ÿr¡   c                 K   sH   t | | dd¡ddgd�}tdddd�}td|| d
œt|f|Ž—Ž}|S )zK Custom ViT base hybrid w/ ResNet50D stride 32. No pretrained weights.
    r9   r   Tr]   r›   r   r€   rŽ   Úvit_base_resnet50d_224r…   )r¢   r    rˆ   r>   r>   r?   r¢   u  s     ÿ ÿ
ÿr¢   rz   r{   r|   r}   ry   )Zvit_tiny_r_s16_p8_224_in21kZvit_small_r26_s32_224_in21kZvit_base_r50_s16_224_in21kZvit_base_resnet50_224_in21kZvit_large_r50_s32_224_in21kZvit_base_resnet50_384)F)r\   )r`   )F)F)F)F)F)F)F)F)F)F)F)F)F)8rL   Ú	functoolsr   Útypingr   r   r   r-   Ztorch.nnr'   Ztorch.nn.functionalZ
functionalrA   Z	timm.datar   r   Ztimm.layersr   r	   r
   r   r   Ú	_registryr   r   r   Zresnetr   r   Zresnetv2r   r   Zvision_transformerr   r   r(   r   rR   r[   ri   ru   Zdefault_cfgsr„   rŠ   r�   r�   r‘   r’   r“   r™   rš   r�   rŸ   r¡   r¢   rI   r>   r>   r>   r?   Ú<module>   s   O 


ü   ýý  ý ýý  ý   ý  ý ý  ý  ÿ  ÿ  ÿ  ÿÃB











ú