U
    ¯Ê¹jö  ã                   @   sx   d Z ddlZddlm  mZ ddlmZ ddlZej	dœdd„Z
ej	dœdd„Zeeed	œd
d„ZG dd„ deƒZdS )a3  
AdamP Optimizer Implementation copied from https://github.com/clovaai/AdamP/blob/master/adamp/adamp.py

Paper: `Slowing Down the Weight Norm Increase in Momentum-based Optimizers` - https://arxiv.org/abs/2006.08217
Code: https://github.com/clovaai/AdamP

Copyright (c) 2020-present NAVER Corp.
MIT license
é    N)Ú	Optimizer)Úreturnc                 C   s   |   |  d¡d¡S )Nr   éÿÿÿÿ)ÚreshapeÚsize©Úx© r	   úO/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/optim/adamp.pyÚ_channel_view   s    r   c                 C   s   |   dd¡S )Né   r   )r   r   r	   r	   r
   Ú_layer_view   s    r   )ÚdeltaÚwd_ratioÚepsc                 C   sÂ   d}ddt | jƒd   }ttfD ]–}|| ƒ}	||ƒ}
tj|
|	d|d� ¡ }| ¡ |t 	|	 
d¡¡ k r"| |	jddd� |¡ |¡ }||||| ƒjdd� |¡ 8 }|}||f  S q"||fS )	Nç      ð?)r   )r   r   )Údimr   é   )Úpr   )r   )ÚlenÚshaper   r   ÚFZcosine_similarityZabs_ÚmaxÚmathÚsqrtr   ZnormÚadd_r   Úsum)r   ÚgradÚperturbr   r   r   ÚwdZexpand_sizeZ	view_funcZ
param_viewZ	grad_viewZ
cosine_simZp_nr	   r	   r
   Ú
projection   s    "r    c                       s0   e Zd Zd‡ fdd„	Ze ¡ dd
d„ƒZ‡  ZS )ÚAdamPçü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   çš™™™™™¹?Fc	           
   	      s,   t |||||||d�}	tt| ƒ ||	¡ d S )N)ÚlrÚbetasr   Úweight_decayr   r   Únesterov)ÚdictÚsuperr!   Ú__init__)
ÚselfÚparamsr&   r'   r   r(   r   r   r)   Údefaults©Ú	__class__r	   r
   r,   ,   s         þzAdamP.__init__Nc              
   C   sÞ  d }|d k	r&t  ¡ � |ƒ }W 5 Q R X | jD �]ª}|d D �]š}|jd krLq:|j}|d \}}|d }| j| }	t|	ƒdkr d|	d< t  |¡|	d< t  |¡|	d< |	d |	d  }
}|	d  d7  < d||	d   }d||	d   }|
 |¡j|d| d	� | |¡j	||d| d
� | 
¡ t 
|¡  |d ¡}|d | }|�r\||
 d| |  | }n|
| }d}t|jƒdk�ršt||||d |d |d ƒ\}}|d dk�rÆ| d|d |d  |  ¡ |j|| d	� q:q,|S )Nr.   r'   r)   r   ÚstepÚexp_avgÚ
exp_avg_sqr   )Úalpha)Úvaluer   r&   r   r   r   r(   )ÚtorchZenable_gradZparam_groupsr   Ústater   Z
zeros_likeZmul_r   Zaddcmul_r   r   r   r    )r-   ÚclosureZlossÚgroupr   r   Zbeta1Zbeta2r)   r8   r3   r4   Zbias_correction1Zbias_correction2ÚdenomZ	step_sizer   r   r	   r	   r
   r2   3   sD    


"z
AdamP.step)r"   r#   r$   r   r%   r%   F)N)Ú__name__Ú
__module__Ú__qualname__r,   r7   Zno_gradr2   Ú__classcell__r	   r	   r0   r
   r!   +   s           ÿr!   )Ú__doc__r7   Ztorch.nn.functionalÚnnZ
functionalr   Ztorch.optim.optimizerr   r   ZTensorr   r   Úfloatr    r!   r	   r	   r	   r
   Ú<module>   s   
