U ʹj= @sdZddlZddlmZddlmZmZmZddlZddl m Z ddl m Z ddl mZddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlmZddlm Z ddl!m"Z"ddl#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.ddl/m0Z0e1e2Z3dhZ4d4e j5dddZ6ddZ7d5d!d"Z8d6e j5e9ee:e9ee9e;d&d'd(Zd8e:ee9e9e9ee;e;ee9eed1d2d3Z?dS)9z\ Optimizer Factory w/ Custom Weight Decay Hacked together by / Copyright 2021 Ross Wightman N)islice)OptionalCallableTuple)group_parameters) AdaBelief) Adafactor) Adahessian)AdamP)Adan)Lamb)Lars)Lion) Lookahead)MADGRAD)Nadam)NAdamW) NvNovoGrad)RAdam) RMSpropTF)SGDP)SGDWlionh㈵>)modelcCspt|}g}g}|D]B\}}|js(q|jdksD|dsD||krP||q||q|dd||dgS)Nrz.bias)params weight_decay)setnamed_parameters requires_gradndimendswithappend)rrno_weight_decay_listdecayno_decaynameparamrrW/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/optim/optim_factory.pyparam_groups_weight_decay*s  r,csttfdddS)NcsttSN)tuplerritsizerr+Bz_group..r)iterr/rr/r+_group@sr5 c sdd}t|didd}g}g}|D]&\}}|||rH||n||q,t|} |dk rp| |  }tt||}t|ddt|D} | fdd|D| S)Ncs:|sdSt|ttfr,tfdd|DS|SdS)NTcsg|]}|qSr) startswith).0Zhpinrr+ Jsz0_layer_map.._in_head..) isinstancer.listanyr7)r:hprr9r+_in_headFs z_layer_map.._in_headZpretrained_cfg classifiercSs i|]\}}|D] }||qqSrr)r8ilr:rrr+ [s z_layer_map..csi|] }|qSrr)r8r:Znum_trunk_groupsrr+rD\s) getattrgetr!r%lenr=r5 enumerateupdate) rZlayers_per_groupZ num_groupsr@Z head_prefixZ names_trunkZ names_headr:_Znum_trunk_layers layer_maprrEr+ _layer_mapEs  rM皙??F)rrr& layer_decayend_layer_decayverbosecsJt|}i}i}t|dr2t||jdddd}nt|}t|d} | dtfddt| D} | D]\} } | j sqv| j dks| |krd } d }nd } |}| | }d || f}||kr| |}||gd ||<||gd||<||d | ||d | qv|r>ddl}td|j|ddt|S)z Parameter groups for layer-wise lr decay & weight decay Based on BEiT: https://github.com/microsoft/unilm/blob/master/beit/optim_factory.py#L58 group_matcherF)ZcoarseT)reverserc3s|]}|VqdSr-r)r8rBrPZ layer_maxrr+ xsz+param_groups_layer_decay..r(rr'z layer_%d_%s)lr_scaler param_names)rWrrrXrrNzparameter groups: %s)indent)r hasattrrrSrMmaxvaluesr=ranger!r"r#rGr%json_loggerinfodumps)rrr&rPrQrRZparam_group_namesZ param_groupsrLZ num_layersZ layer_scalesr)r*Zg_decayZ this_decayZlayer_id group_nameZ this_scaler_rrUr+param_groups_layer_decay`sF      rdcCst|j|j|j|jd}t|dddk r2|j|d<t|dddk rL|j|d<t|dddk rf|j|d<t|dddk r| |j t|d ddk r|j |d <|S) z cfg/argparse to kwargs helper Convert optimizer args in argparse args or cfg like object to keyword args for updated create fn. )optlrrmomentumopt_epsNeps opt_betasbetasrPopt_args opt_foreachforeach) dictrerfrrgrFrhrjrPrJrlrm)cfgkwargsrrr+optimizer_kwargss"     rrTcCst|ft|dd|iS)zk Legacy optimizer factory for backwards compatibility. NOTE: Use create_optimizer_v2 for new code. )rpfilter_bias_and_bn)create_optimizer_v2rr)argsrrsrrr+create_optimizersrvsgdr?)rerfrrgrnrsrPparam_group_fnc Ks& t|tjrri} t|dr"|} |r0||} qv|dk rNt|||| d} d}qv|rh|rht||| } d}qv|} n|} |} | d} | d} | drz dd l m }m }m}m}d }Wntk rd }YnX|rtjstd | d rt*| fd,d i|}n| d-kr`t*| fd,d i|}n| d.krzt+| f|}n| d/krt+| fd0d i|}nb| d1krt,| f|d d2|}n>| d3krt,| fd|i|}n| d4krt,| f|d d d5|}n| d6kr,t,| f|d d|}n| d7krNt-| fd|i|}n| d8krrt-| f|d d9|}n| d:ks| d;krt.| f|}nh| d|}nB| d?krt0| fd=|d>|}n| d@kr|ddt1| f|}n| dAkr t2| f|}n| dBkrP|dd|| f|d d|}n| dCkr|dd|| f|d d|}n~| dDkr|| fdEd i|}n\| dFkr|| fdEd i|}n:| dGkr|| f|}n | dHkr|dIdJ|| f|}n| dKkr8|dd|jj| f|d d|}n| dLkrl|dd|jj3| f|d d|}n| dMkr|dd|jj| fd|i|}n`| dNkr|dd|jj3| fd|i|}n.| dOkr|jj| f|}n| dPkr |jj4| f|}n| dQkr&|jj| f|}n| dRkrB|jj5| f|}n| dSkr^|jj6| f|}n| dTkrz|jj7| f|}n| dUkr|jj8| f|}nh| dVkr|jj7| f|}nL| dWkr|jj1| f|}n0| dXkr|jj9| f|}nd rdYstt:t;| dZk r"| dd[k r"t<|}|S)\a Create an optimizer. TODO currently the model is passed in and all parameters are selected for optimization. For more general use an interface that allows selection of parameters to optimize and lr groups, one of: * a filter fn interface that further breaks params into groups in a weight_decay compatible fashion * expose the parameters interface and leave it up to caller Args: model_or_params (nn.Module): model containing parameters to optimize opt: name of optimizer to create lr: initial learning rate weight_decay: weight decay to apply in optimizer momentum: momentum for momentum based optimizers (others may use betas via kwargs) foreach: Enable / disable foreach (multi-tensor) operation if True / False. Choose safe default if None filter_bias_and_bn: filter out bias, bn and other 1d params from weight decay **kwargs: extra optimizer specific kwargs to pass through Returns: Optimizer no_weight_decayN)rrPr&rrKZfusedr) FusedNovoGrad FusedAdam FusedLAMBFusedSGDTFz+APEX and CUDA required for fused optimizersbnbz1bitsandbytes and CUDA required for bnb optimizersrrfrnrwnesterovri)rgrrgsgdpsgdwZ nesterovwZ momentumwZadamZadamwadampg{Gz?)Zwd_ratiornadamnadamwradamZadamax adabeliefZrectifyZ radabeliefZadadeltaZadagradg:0yE> adafactorZadanpZno_proxZadanwlambZlambc trust_clipZlarc)rgrlarsZnlarc)rgrrZnlarsmadgradZmadgradw)rgZdecoupled_decayZnovograd nvnovogradZrmsproprx)alphargZ rmsproptfr adahessianZfusedsgdZ fusedmomentumZ fusedadamZ adam_w_modeZ fusedadamwZ fusedlambZ fusednovogradrk)gffffff?g\(\?ZbnbsgdZ bnbsgd8bitZ bnbmomentumZbnbmomentum8bitZbnbadamZ bnbadam8bitZbnbadamwZ bnbadamw8bitZbnblambZ bnblamb8bitZbnblarsZ bnblarsb8bitZbnblionZ bnblion8bitzInvalid optimizerr lookahead)=r<nnModuler[rzrdr, parameterslowersplitr7Zapex.optimizersr|r}r~r ImportErrortorchcudaZ is_availableAssertionErrorZ bitsandbytesro setdefault_DEFAULT_FOREACHpopoptimZSGDrrZAdamZAdamWr rAttributeErrorrrZAdamaxrZAdadeltaZAdagradr r r rrrZRMSproprrr ZSGD8bitZAdam8bitZ AdamW8bitZLAMBZLAMB8bitZLARSZLion8bit ValueErrorrHr)Zmodel_or_paramsrerfrrgrnrsrPryrqrzrZ opt_lowerZ opt_splitr|r}r~rZhas_apexrZhas_bnbrlZ optimizerrrr+rtsR                                                                            rt)rr)r6N)rNrrONF)T)rwNrrxNTNN)@__doc__logging itertoolsrtypingrrrrZtorch.nnrZ torch.optimrZ timm.modelsrrrrr rr rr Zadanr rr rrrrrrrrrrrrrrrrZ rmsprop_tfrrrrr getLogger__name__r`rrr,r5rMfloatstrboolrdrrrvrtrrrr+s                         @