U
    ¯Ê¹jŒ  ã                   @   s4   d Z ddlZddlZddlmZ G dd„ deƒZdS )zºRAdam Optimizer.
Implementation lifted from: https://github.com/LiyuanLucasLiu/RAdam
Paper: `On the Variance of the Adaptive Learning Rate and Beyond` - https://arxiv.org/abs/1908.03265
é    N)Ú	Optimizerc                       s<   e Zd Zd‡ fdd„	Z‡ fdd„Ze ¡ dd
d„ƒZ‡  ZS )ÚRAdamçü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   c                    s6   t ||||dd„ tdƒD ƒd�}tt| ƒ ||¡ d S )Nc                 S   s   g | ]}d d d g‘qS ©N© )Ú.0Ú_r   r   úO/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/optim/radam.pyÚ
<listcomp>   s     z"RAdam.__init__.<locals>.<listcomp>é
   )ÚlrÚbetasÚepsÚweight_decayÚbuffer)ÚdictÚrangeÚsuperr   Ú__init__)ÚselfÚparamsr   r   r   r   Údefaults©Ú	__class__r   r   r      s       þzRAdam.__init__c                    s   t t| ƒ |¡ d S r   )r   r   Ú__setstate__)r   Ústater   r   r   r      s    zRAdam.__setstate__Nc              	   C   s¬  d }|d k	r&t  ¡ � |ƒ }W 5 Q R X | jD �]x}|d D �]h}|jd krLq:|j ¡ }|jrdtdƒ‚| ¡ }| j| }t|ƒdkr¨d|d< t  	|¡|d< t  	|¡|d< n$|d  
|¡|d< |d  
|¡|d< |d |d  }}	|d \}
}|	 |¡j||d| d	� | |
¡j|d|
 d
� |d  d7  < |d t|d d ƒ }|d |d k�rj|d |d  }}nÆ|d |d< ||d  }dd|  d }|d|d  | d|   }||d< |dk�r|d t d| |d  |d  |d  | | |d  ¡ d|
|d    }n|d d|
|d    }||d< |d dk�rZ|j||d  |d  d
� |dk�rŠ|	 ¡  |d ¡}|j||| d	� n|j|| d
� | |¡ q:q,|S )Nr   z'RAdam does not support sparse gradientsr   ÚstepÚexp_avgÚ
exp_avg_sqr   é   )Úvalue)Úalphar   r   é   é   r   é   r   r   )ÚtorchZenable_gradZparam_groupsÚgradÚfloatZ	is_sparseÚRuntimeErrorr   ÚlenZ
zeros_likeZtype_asZmul_Zaddcmul_Zadd_ÚintÚmathÚsqrtZaddcdiv_Zcopy_)r   ÚclosureZlossÚgroupÚpr(   Zp_fp32r   r   r    Zbeta1Zbeta2ÚbufferedZnum_smaZ	step_sizeZbeta2_tZnum_sma_maxÚdenomr   r   r   r      sx    





ÿÿþþýýÿü
z
RAdam.step)r   r   r   r   )N)	Ú__name__Ú
__module__Ú__qualname__r   r   r'   Zno_gradr   Ú__classcell__r   r   r   r   r   
   s   r   )Ú__doc__r-   r'   Ztorch.optim.optimizerr   r   r   r   r   r   Ú<module>   s   