U
    ¯Ê¹jø  ã                   @   sV   d Z ddlZddlm  mZ ddlmZmZ ddl	Z	ddl
mZ G dd„ deƒZdS )a1  
SGDP Optimizer Implementation copied from https://github.com/clovaai/AdamP/blob/master/adamp/sgdp.py

Paper: `Slowing Down the Weight Norm Increase in Momentum-based Optimizers` - https://arxiv.org/abs/2006.08217
Code: https://github.com/clovaai/AdamP

Copyright (c) 2020-present NAVER Corp.
MIT license
é    N)Ú	OptimizerÚrequiredé   )Ú
projectionc                       s@   e Zd Zedddddddf‡ fdd„	Ze ¡ d
dd	„ƒZ‡  ZS )ÚSGDPr   Fg:Œ0âŽyE>gš™™™™™¹?c
              
      s.   t ||||||||	d�}
tt| ƒ ||
¡ d S )N)ÚlrÚmomentumÚ	dampeningÚweight_decayÚnesterovÚepsÚdeltaÚwd_ratio)ÚdictÚsuperr   Ú__init__)ÚselfÚparamsr   r   r	   r
   r   r   r   r   Údefaults©Ú	__class__© úN/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/optim/sgdp.pyr      s          þzSGDP.__init__Nc              
   C   sJ  d }|d k	r&t  ¡ � |ƒ }W 5 Q R X | jD �]}|d }|d }|d }|d }|d D ]è}|jd krjqZ|j}	| j| }
t|
ƒdkr”t  |¡|
d< |
d }| |¡j|	d| d� |rÆ|	||  }n|}d}t|j	ƒd	krþt
||	||d
 |d |d ƒ\}}|dk�r.| d|d |d  | d	|   ¡ |j||d  d� qZq,|S )Nr
   r   r	   r   r   r   g      ð?)Úalphar   r   r   r   r   )ÚtorchZenable_gradZparam_groupsÚgradÚstateÚlenZ
zeros_likeZmul_Zadd_Úshaper   )r   ÚclosureZlossÚgroupr
   r   r	   r   Úpr   r   ÚbufZd_pr   r   r   r   Ústep   s8    


"
&z	SGDP.step)N)	Ú__name__Ú
__module__Ú__qualname__r   r   r   Zno_gradr#   Ú__classcell__r   r   r   r   r      s       ÿr   )Ú__doc__r   Ztorch.nn.functionalÚnnZ
functionalÚFZtorch.optim.optimizerr   r   ÚmathZadampr   r   r   r   r   r   Ú<module>   s   
