U
    ¯Ê¹jY  ã                   @   s0   d dl Z d dlZd dlmZ G dd„ deƒZdS )é    N)Ú	Optimizerc                       s4   e Zd ZdZd‡ fdd„	Ze ¡ dd
d„ƒZ‡  ZS )ÚNadamaþ  Implements Nadam algorithm (a variant of Adam based on Nesterov momentum).

    It has been proposed in `Incorporating Nesterov Momentum into Adam`__.

    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 2e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        schedule_decay (float, optional): momentum schedule decay (default: 4e-3)

    __ http://cs229.stanford.edu/proj2015/054_report.pdf
    __ http://www.cs.toronto.edu/~fritz/absps/momentum.pdf

        Originally taken from: https://github.com/pytorch/pytorch/pull/1408
        NOTE: Has potential issues but does work well on some problems.
    çü©ñÒMb`?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   çü©ñÒMbp?c                    s>   d|kst d |¡ƒ‚t|||||d�}tt| ƒ ||¡ d S )Ng        zInvalid learning rate: {})ÚlrÚbetasÚepsÚweight_decayÚschedule_decay)Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)ÚselfÚparamsr   r	   r
   r   r   Údefaults©Ú	__class__© úO/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/optim/nadam.pyr      s    ûzNadam.__init__Nc              	   C   sø  d}|dk	r&t  ¡ � |ƒ }W 5 Q R X | jD �]Ä}|d D �]´}|jdkrLq:|j}| j| }t|ƒdkr”d|d< d|d< t  |¡|d< t  |¡|d< |d }|d	 }|d |d  }	}
|d
 \}}|d }|d  d7  < |d }d||  }|d dk�r|j||d d�}|ddd||     }|ddd|d |     }|| }|| | }||d< |	 |¡j	|d| d� |
 |¡j
||d| d� |
 ¡ t |¡  	|¡}|j|||d  d|  d|  d� |j|	||d  | d|  d� q:q,|S )z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   Ústepg      ð?Ú
m_scheduleÚexp_avgÚ
exp_avg_sqr   r	   r
   é   r   )Úalphag      à?g¸…ëQ¸î?)Úvaluer   )ÚtorchZenable_gradZparam_groupsÚgradÚstateÚlenZ
zeros_likeÚaddZmul_Zadd_Zaddcmul_ÚsqrtÚmathZaddcdiv_)r   ÚclosureZlossÚgroupÚpr!   r"   r   r   r   r   Zbeta1Zbeta2r
   ÚtZbias_correction2Zmomentum_cache_tZmomentum_cache_t_1Zm_schedule_newZm_schedule_nextÚdenomr   r   r   r   +   sH    


&&z
Nadam.step)r   r   r   r   r   )N)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r    Zno_gradr   Ú__classcell__r   r   r   r   r      s       ÿr   )r&   r    Ztorch.optim.optimizerr   r   r   r   r   r   Ú<module>   s   