U ʹj^@sdZddlZddlmZddlZddlmZddlmZm Z ddl m Z m Z m Z mZmZmZmZddlmZddlmZmZdd lmZmZmZd d gZGd d d ejZGd ddejZGdddejZGdddejZ GdddejZ!Gdd d ejZ"dleje#e$dddZ%ddZ&dmddZ'dnd d!Z(ee(e(e(e(d"d#d$e(d"d%d&d'e(e(d"d(d$e(d"d)d&d'e(d"d*d+d,d-d.d/d0e(d"d1d+d,d-d.d2e(ee d3e(d"d4ee d5e(d"d6ee d5e(d"d7ee d5e(d"d8ee d5e(d"d9ee d5e(d"d:ee d5e(d"d;ee d5e(d"dee d5e(d"d?ee d5e(d"d@ee d5e(e(d"dAd$e(dBZ)edoe"dCdDdEZ*edpe"dCdFdGZ+edqe"dCdHdIZ,edre"dCdJdKZ-edse"dCdLdMZ.edte"dCdNdOZ/edue"dCdPdQZ0edve"dCdRdSZ1edwe"dCdTdUZ2edxe"dCdVdWZ3edye"dCdXdYZ4edze"dCdZd[Z5ed{e"dCd\d]Z6ed|e"dCd^d_Z7ed}e"dCd`daZ8ee9dbdcdddedfdgdhdidjdUdWdk dS)~uj MLP-Mixer, ResMLP, and gMLP in PyTorch This impl originally based on MLP-Mixer paper. Official JAX impl: https://github.com/google-research/vision_transformer/blob/linen/vit_jax/models_mixer.py Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 @article{tolstikhin2021, title={MLP-Mixer: An all-MLP Architecture for Vision}, author={Tolstikhin, Ilya and Houlsby, Neil and Kolesnikov, Alexander and Beyer, Lucas and Zhai, Xiaohua and Unterthiner, Thomas and Yung, Jessica and Keysers, Daniel and Uszkoreit, Jakob and Lucic, Mario and Dosovitskiy, Alexey}, journal={arXiv preprint arXiv:2105.01601}, year={2021} } Also supporting ResMlp, and a preliminary (not verified) implementations of gMLP Code: https://github.com/facebookresearch/deit Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404 @misc{touvron2021resmlp, title={ResMLP: Feedforward networks for image classification with data-efficient training}, author={Hugo Touvron and Piotr Bojanowski and Mathilde Caron and Matthieu Cord and Alaaeldin El-Nouby and Edouard Grave and Armand Joulin and Gabriel Synnaeve and Jakob Verbeek and Hervé Jégou}, year={2021}, eprint={2105.03404}, } Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050 @misc{liu2021pay, title={Pay Attention to MLPs}, author={Hanxiao Liu and Zihang Dai and David R. So and Quoc V. Le}, year={2021}, eprint={2105.08050}, } A thank you to paper authors for releasing code and weights. Hacked together by / Copyright 2021 Ross Wightman N)partial)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD) PatchEmbedMlpGluMlpGatedMlpDropPath lecun_normal_ to_2tuple)build_model_with_cfg) named_applycheckpoint_seq)generate_default_cfgsregister_modelregister_model_deprecations MixerBlockMlpMixercsBeZdZdZdeeejddejddffdd Z dd Z Z S) rz Residual Block w/ token mixing and channel MLPs Based on: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 ?@ư>epsc sztfddt|D\} } ||_||| ||d|_|dkrPt|nt|_||_ || ||d|_ dS)Ncsg|]}t|qS)int).0xdimrT/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/models/mlp_mixer.py Hsz'MixerBlock.__init__.. act_layerdropr) super__init__r norm1 mlp_tokensr nnIdentity drop_pathnorm2 mlp_channels) selfr!seq_len mlp_ratio mlp_layer norm_layerr%r&r-Z tokens_dimZ channels_dim __class__r r"r(<s   zMixerBlock.__init__c CsH|||||dddd}|||||}|SNr )r-r*r) transposer/r.r0rrrr"forwardOs*zMixerBlock.forward) __name__ __module__ __qualname____doc__rrr+ LayerNormGELUr(r; __classcell__rrr5r"r8s cs$eZdZfddZddZZS)Affinecs>tttdd|f|_ttdd|f|_dS)Nr ) r'r(r+ ParametertorchonesalphaZzerosbeta)r0r!r5rr"r(Vs zAffine.__init__cCst|j|j|SN)rEZaddcmulrHrGr:rrr"r;[szAffine.forward)r<r=r>r(r;rBrrr5r"rCUs rCcs:eZdZdZdeeejdddffdd ZddZ Z S) ResBlockz Residual MLP block w/ LayerScale and Affine 'norm' Based on: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404 g-C6?rc stt||} |||_t|||_| dkr>t| nt|_ |||_ ||| ||d|_ t |t ||_t |t ||_dS)Nrr$)r'r(rr)r+Linear linear_tokensr r,r-r.r/rDrErFls1ls2) r0r!r1r2r3r4r% init_valuesr&r- channel_dimr5rr"r(ds    zResBlock.__init__c CsT|||j|||dddd}|||j|||}|Sr7)r-rNrMr)r9rOr/r.r:rrr"r;zs0 zResBlock.forward) r<r=r>r?rrCr+rAr(r;rBrrr5r"rJ_srJcs6eZdZdZejffdd ZddZddZZ S)SpatialGatingUnitzc Spatial Gating Unit Based on: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050 cs.t|d}|||_t|||_dS)Nr8)r'r(normr+rLproj)r0r!r1r4Zgate_dimr5rr"r(s  zSpatialGatingUnit.__init__cCs(tjj|jjddtj|jjdS)Nrstd)r+initnormal_rTweightones_biasr0rrr" init_weightsszSpatialGatingUnit.init_weightscCs>|jddd\}}||}||dd}||ddS)Nr8r )chunkrSrTr9)r0ruvrrr"r;s zSpatialGatingUnit.forward) r<r=r>r?r+r@r(r]r;rBrrr5r"rRsrRcsBeZdZdZdeeejddejddffdd Z dd Z Z S) SpatialGatingBlockzp Residual Block w/ Spatial Gating Based on: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050 rKrrrc s^tt||} |||_tt|d} ||| || |d|_|dkrPt|nt |_ dS)N)r1)r%Z gate_layerr&r) r'r(rrSrrRr/r r+r,r-) r0r!r1r2r3r4r%r&r-rQZsgur5rr"r(s    zSpatialGatingBlock.__init__cCs|||||}|SrI)r-r/rSr:rrr"r;szSpatialGatingBlock.forward) r<r=r>r?rrr+r@rAr(r;rBrrr5r"rcs rccseZdZdddddddeeeejdd ejd d d d d d ffd d Z e j j d"ddZ e j j d#ddZe j j d$ddZe j j ddZd%ddZddZd&edddZd d!ZZS)'rrrrrFavgc st|_|___d_t||||r:ndd_t j f ddt |D_ _ t | _|dkrt jnt _j|ddS)NF)img_size patch_sizein_chans embed_dimr4c s(g|] }jjdqS))r3r4r%r&r-)stemZ num_patches)r_ r% block_layerdrop_path_raternr3r2r4proj_drop_rater0rr"r#s z%MlpMixer.__init__..r)nlhb)r'r( num_classes global_poolZ num_featuresrngrad_checkpointingrror+Z SequentialrangeblocksrSZDropout head_droprLr,headr])r0rvrkrmrl num_blocksrnr2rrr3r4r%Z drop_ratertrsruZ stem_normrwr5rqr"r(s&       zMlpMixer.__init__cCs.|rt|j nd}ttt|d|ddS)Nr) head_bias)module)mathlogrvrr _init_weights)r0rur~rrr"r]szMlpMixer.init_weightscCstdddgdS)Nz^stem)z^blocks\.(\d+)N)z^norm)i)rorz)dict)r0Zcoarserrr" group_matcherszMlpMixer.group_matcherTcCs ||_dSrI)rx)r0enablerrr"set_grad_checkpointingszMlpMixer.set_grad_checkpointingcCs|jSrI)r|r\rrr"get_classifierszMlpMixer.get_classifierNcCsD||_|dk r |dkst||_|dkr6t|j|nt|_dS)N)rjr)rvAssertionErrorrwr+rLrnr,r|)r0rvrwrrr"reset_classifiers  zMlpMixer.reset_classifiercCs@||}|jr(tjs(t|j|}n ||}||}|SrI)rorxrEjitZ is_scriptingrrzrSr:rrr"forward_featuress    zMlpMixer.forward_features) pre_logitscCs2|jdkr|jdd}||}|r(|S||S)Nrjr r )rwmeanr{r|)r0rrrrr" forward_head s   zMlpMixer.forward_headcCs||}||}|SrI)rrr:rrr"r;s  zMlpMixer.forward)F)F)T)N)F)r<r=r>rrrr+r@rAr(rErignorer]rrrrrboolrr;rBrrr5r"rs< 4      rF)rnamer~cCs t|tjr|dr6tj|jtj|j|nj|r^t |j|jdk rtj|jnBtj |j|jdk rd|krtjj |jddntj|jnzt|tj rt |j|jdk rtj|jnJt|tj tjtjfrtj|jtj|jnt|dr|dS)zA Mixer weight initialization (trying to match Flax defaults) r|NZmlprrUr]) isinstancer+rL startswithrWZzeros_rYZ constant_r[r Zxavier_uniform_rXZConv2dr@Z BatchNorm2dZ GroupNormrZhasattrr])rrr~Zflaxrrr"rs,         rcCsd|kr|i}|D]b\}}|dd}|dd}|dd}|dd }|d s`|d rn|d d d }|||<q|S|S)z Remap checkpoints if needed zpatch_embed.proj.weightz patch_embed.zstem.zattn.zlinear_tokens.zmlp.z mlp_channels.Zgamma_Zlsz.alphaz.betar r^)itemsreplaceendswithZreshape)Z state_dictmodelZout_dictkrbrrr"checkpoint_filter_fn;s     rcKs0|ddrtdtt||fdti|}|S)NZ features_onlyz3features_only not implemented for MLP-Mixer models.Zpretrained_filter_fn)get RuntimeErrorr rr)variant pretrainedkwargsrrrr" _create_mixerLs rrc Ks |ddddddddddd |S) Nrd)rfrere?ZbicubicT)rrrz stem.projr|) urlrvZ input_sizeZ pool_sizecrop_pct interpolationZfixed_input_sizerrVZ first_conv classifierr)rrrrr"_cfgZsrztimm/zlhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_b16_224-76587d61.pth) hf_hub_idrzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_b16_224_in21k-617b3de2.pthiSU)rrrvzlhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_l16_224-92f9adc4.pthzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_l16_224_in21k-846aa33c.pthzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/mixer_b16_224_miil_in21k-2a558a71.pth)rrr)?rrrZbilineari+)rrrrVrrrvzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/mixer_b16_224_miil-9229a591.pth)rrrrVrr)rrVzohttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/gmixer_24_224_raa-7daf7ae6.pth)rrrrVz9https://dl.fbaipublicfiles.com/deit/resmlp_12_no_dist.pthz9https://dl.fbaipublicfiles.com/deit/resmlp_24_no_dist.pthz9https://dl.fbaipublicfiles.com/deit/resmlp_36_no_dist.pthz:https://dl.fbaipublicfiles.com/deit/resmlpB_24_no_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_12_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_24_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_36_dist.pthz7https://dl.fbaipublicfiles.com/deit/resmlpB_24_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlpB_24_22k.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_12_dino.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_24_dino.pthznhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/gmlp_s16_224_raa-10536d42.pth)zmixer_s32_224.untrainedzmixer_s16_224.untrainedzmixer_b32_224.untrained mixer_b16_224.goog_in21k_ft_in1kzmixer_b16_224.goog_in21kzmixer_l32_224.untrained mixer_l16_224.goog_in21k_ft_in1kzmixer_l16_224.goog_in21kmixer_b16_224.miil_in21k mixer_b16_224.miil_in21k_ft_in1kzgmixer_12_224.untrainedzgmixer_24_224.ra3_in1kzresmlp_12_224.fb_in1kzresmlp_24_224.fb_in1kzresmlp_36_224.fb_in1kzresmlp_big_24_224.fb_in1kresmlp_12_224.fb_distilled_in1kresmlp_24_224.fb_distilled_in1kresmlp_36_224.fb_distilled_in1k#resmlp_big_24_224.fb_distilled_in1k"resmlp_big_24_224.fb_in22k_ft_in1kzresmlp_12_224.fb_dinozresmlp_24_224.fb_dinozgmlp_ti16_224.untrainedzgmlp_s16_224.ra3_in1kzgmlp_b16_224.untrained)returncKs,tfdddd|}tdd|i|}|S)zv Mixer-S/32 224x224 Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 rhrirlr}rn mixer_s32_224r)rrrrrZ model_argsrrrr"rsrcKs,tfdddd|}tdd|i|}|S)zw Mixer-S/16 224x224 Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 rgrhrir mixer_s16_224r)rrrrrr"rsrcKs,tfdddd|}tdd|i|}|S)zw Mixer-B/32 224x224 Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 r r mixer_b32_224r)rrrrrr"rsrcKs,tfdddd|}tdd|i|}|S)z Mixer-B/16 224x224. ImageNet-1k pretrained weights. Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 rgrrr mixer_b16_224r)rrrrrr"rsrcKs,tfdddd|}tdd|i|}|S)zx Mixer-L/32 224x224. Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 rr mixer_l32_224r)rrrrrr"rsrcKs,tfdddd|}tdd|i|}|S)z Mixer-L/16 224x224. ImageNet-1k pretrained weights. Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 rgrrr mixer_l16_224r)rrrrrr"rsrc Ks4tfddddttjd|}tdd|i|}|S) zV Glu-Mixer-12 224x224 Experiment by Ross Wightman, adding SwiGLU to MLP-Mixer rgrrrrlr}rnr2r3r% gmixer_12_224r)rrrr+ZSiLUrrrrr"rsrc Ks4tfddddttjd|}tdd|i|}|S) zV Glu-Mixer-24 224x224 Experiment by Ross Wightman, adding SwiGLU to MLP-Mixer rgrrrr gmixer_24_224r)rrrrrr"rsrc Ks2tfddddttd|}tdd|i|}|S) zx ResMLP-12 Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404 rgrrrKrlr}rnr2rrr4 resmlp_12_224r)r)rrJrCrrrrr"rsrc Ks:tfddddttddtd|}td d |i|}|S) zx ResMLP-24 Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404 rgrrrKgh㈵>rPr resmlp_24_224r)rrrrJrCrrrrr"r(s rc Ks:tfddddttddtd|}td d |i|}|S) zx ResMLP-36 Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404 rg$rrKrrr resmlp_36_224r)rrrrrr"r4s rc Ks:tfddddttddtd|}td d |i|}|S) zz ResMLP-B-24 Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404 rhrrrKrrrresmlp_big_24_224r)rrrrrr"r@s rc Ks2tfddddttd|}tdd|i|}|S) zU gMLP-Tiny Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050 rgrlr}rnr2rrr3 gmlp_ti16_224r)rrrcrrrrrr"rLsrc Ks2tfddddttd|}tdd|i|}|S) zV gMLP-Small Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050 rgrrr gmlp_s16_224r)rrrrrr"rXsrc Ks2tfddddttd|}tdd|i|}|S) zU gMLP-Base Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050 rgrrirr gmlp_b16_224r)rrrrrr"rdsrrrrrrrrrr) Zmixer_b16_224_in21kZmixer_l16_224_in21kZmixer_b16_224_miilZmixer_b16_224_miil_in21kZresmlp_12_distilled_224Zresmlp_24_distilled_224Zresmlp_36_distilled_224Zresmlp_big_24_distilled_224Zresmlp_big_24_224_in22ft1kZresmlp_12_224_dinoZresmlp_24_224_dino)rF)F)r)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F):r?r functoolsrrEZtorch.nnr+Z timm.datarrZ timm.layersrrrrr r r Z_builderr Z _manipulaterr _registryrrr__all__ModulerrCrJrRrcrstrfloatrrrrZ default_cfgsrrrrrrrrrrrrrrrr<rrrr"sn(  $  !f"   d