U
    ¯Ê¹jQ0  ã                   @   sú   d Z ddlZddlmZmZ ddlZddlmZ G dd„ dejjƒZ	dee ee ee ee ee ee
 e
eeeeee
ddœd	d
„Zee ee ee ee ee eeeeee
e
dœdd„Zee ee ee ee ee eeeeee
e
dœdd„ZdS )z§ NAdamW Optimizer

Based on simplified algorithm in https://github.com/mlcommons/algorithmic-efficiency/tree/main/baselines/nadamw

Added multi-tensor (foreach) path.
é    N)ÚListÚOptional)ÚTensorc                       sN   e Zd ZdZdeee edœ‡ fd	d
„Z‡ fdd„Ze 	¡ ddd„ƒZ
‡  ZS )ÚNAdamWa?  Implements NAdamW algorithm.

      See Table 1 in https://arxiv.org/abs/1910.05446 for the implementation of
      the NAdam algorithm (there is also a comment in the code which highlights
      the only difference of NAdamW and AdamW).
      For further details regarding the algorithm we refer to
      `Decoupled Weight Decay Regularization`_.

      Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay coefficient (default: 1e-2)
      .. _Decoupled Weight Decay Regularization:
          https://arxiv.org/abs/1711.05101
      .. _On the Convergence of Adam and Beyond:
          https://openreview.net/forum?id=ryQu7f-RZ
    çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>ç{®Gáz„?FN)ÚmaximizeÚforeachÚ
capturablec	           
   	      sÂ   d|kst d|› �ƒ‚d|ks,t d|› �ƒ‚d|d   krDdk sXn t d|d › �ƒ‚d|d   krpdk s„n t d|d › �ƒ‚d|ksšt d	|› �ƒ‚t|||||||d
�}	tƒ  ||	¡ d S )Nç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: é   z#Invalid beta parameter at index 1: zInvalid weight_decay value: )ÚlrÚbetasÚepsÚweight_decayr   r
   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)
ÚselfÚparamsr   r   r   r   r
   r   r   Údefaults©Ú	__class__© úP/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/optim/nadamw.pyr   '   s(    ù	zNAdamW.__init__c                    s`   t ƒ  |¡ t| j ¡ ƒ}t|ƒdko6t |d d ¡}|s\|D ]}t t	|d ƒ¡|d< q@d S )Nr   Ústep)
r   Ú__setstate__ÚlistÚstateÚvaluesÚlenÚtorchZ	is_tensorÚtensorÚfloat)r   r"   Zstate_valuesZstep_is_tensorÚsr   r   r   r    G   s    
ÿzNAdamW.__setstate__c                 C   sJ  |   ¡  d}|dk	r.t ¡ � |ƒ }W 5 Q R X | jD �]}g }g }g }g }g }|d \}	}
|d D ]¬}|jdkrrqb| |¡ |jjrŒtdƒ‚| |j¡ | j| }t	|ƒdkrät 
d¡|d< tj|tjd�|d	< tj|tjd�|d
< | |d	 ¡ | |d
 ¡ | |d ¡ qbt||||||	|
|d |d |d |d |d d� q4|S )z¹Performs a single optimization step.

            Args:
              closure (callable, optional): A closure that reevaluates the model
                  and returns the loss.
        Nr   r   z(NAdamW does not support sparse gradientsr   r   r   )Zmemory_formatÚexp_avgÚ
exp_avg_sqr   r   r   r
   r   ©Úbeta1Úbeta2r   r   r   r
   r   )Z _cuda_graph_capture_health_checkr%   Zenable_gradZparam_groupsÚgradÚappendZ	is_sparseÚRuntimeErrorr"   r$   r&   Z
zeros_likeZpreserve_formatÚnadamw)r   ÚclosureZlossÚgroupZparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚstate_stepsr,   r-   Úpr"   r   r   r   r   P   sT    



ôzNAdamW.step)r   r   r   r	   FNF)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úboolr   r   r    r%   Zno_gradr   Ú__classcell__r   r   r   r   r      s           ÷÷ 	r   F)r   r4   r5   r6   r7   r   r   r,   r-   r   r   r   r
   Úreturnc                C   sb   t dd„ |D ƒƒstdƒ‚|dkr&d}|r:tj ¡ s:t}nt}|| |||||||	|
|||d� dS )zcFunctional API that performs NAdamW algorithm computation.
      See NAdamW class for details.
    c                 s   s   | ]}t |tjƒV  qd S ©N)Ú
isinstancer%   r   )Ú.0Útr   r   r   Ú	<genexpr>£   s     znadamw.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNTr+   )Úallr0   r%   ZjitZis_scriptingÚ_multi_tensor_nadamwÚ_single_tensor_nadamw)r   r4   r5   r6   r7   r   r   r,   r-   r   r   r   r
   Úfuncr   r   r   r1   �   s.    ÿôr1   )r   r4   r5   r6   r7   r,   r-   r   r   r   r
   r   c                C   s‚  t | ƒD �]r\}}|
s|| n||  }|| }|| }|| }|d7 }| d||  ¡ | |¡j|d| d� | |¡j||d| d� |�r|}dt ||¡ }dt ||¡ }|| }| ¡ }| ¡ }| |¡j|d| d�}| ¡ ||   |	| ¡}| 	||¡ q| 
¡ }d||  }d||  }|| }t |¡}| |¡j|d| d�}| ¡ |  |	¡}|j	||| d� qd S )Nr   r   ©Úalpha)Úvalue)Ú	enumerateZmul_Zadd_Zaddcmul_r%   ÚpowÚnegÚsqrtÚmulZaddcdiv_ÚitemÚmath)r   r4   r5   r6   r7   r,   r-   r   r   r   r
   r   ÚiÚparamr.   r)   r*   Zstep_tr   Úbias_correction1Úbias_correction2Ú	step_sizeZstep_size_negÚbias_correction2_sqrtÚdenomr   r   r   rG   ¿   s6    
rG   c                   sh  t | ƒdkrd S |r4tdd„ t| |ƒD ƒƒs4tdƒ‚|
rFt t|ƒ¡}dd„ |D ƒ}dd„ |D ƒ}dd„ |D ƒ}d	d„ | D ƒ} t |d
¡ t | d
ˆ|  ¡ t |ˆ ¡ tj||d
ˆ  d� t |ˆ¡ t 	|||d
ˆ ¡ |�rÎ‡ fdd„|D ƒ}‡fdd„|D ƒ}t 
|d
¡ t 
|d
¡ t |¡ t |¡ t |ˆ¡}t |¡ t |¡ t |¡}t |ˆ ¡}tj||d
ˆ  d� t |¡}t |t ||¡¡ t ||	¡}t |¡ t ||¡}t | ||¡ n–‡ fdd„|D ƒ}‡fdd„|D ƒ}‡fdd„|D ƒ}dd„ |D ƒ}t |ˆ ¡}tj||d
ˆ  d� t |¡}t ||¡ t ||	¡}t | |||¡ d S )Nr   c                 s   s   | ]\}}|j o|j V  qd S r@   )Zis_cuda)rB   r8   r   r   r   r   rD     s    z'_multi_tensor_nadamw.<locals>.<genexpr>z@If capturable=True, params and state_steps must be CUDA tensors.c                 S   s$   g | ]}t  |¡rt  |¡n|‘qS r   ©r%   Z
is_complexZview_as_real©rB   Úxr   r   r   Ú
<listcomp>  s     z(_multi_tensor_nadamw.<locals>.<listcomp>c                 S   s$   g | ]}t  |¡rt  |¡n|‘qS r   rZ   r[   r   r   r   r]     s     c                 S   s$   g | ]}t  |¡rt  |¡n|‘qS r   rZ   r[   r   r   r   r]     s     c                 S   s$   g | ]}t  |¡rt  |¡n|‘qS r   rZ   r[   r   r   r   r]     s     r   rI   c                    s   g | ]}t  ˆ |¡‘qS r   ©r%   rM   ©rB   r   ©r,   r   r   r]   /  s     c                    s   g | ]}t  ˆ |¡‘qS r   r^   r_   ©r-   r   r   r]   0  s     c                    s   g | ]}d ˆ |  ¡   ‘qS ©r   ©rQ   r_   r`   r   r   r]   M  s     c                    s   g | ]}d ˆ |  ¡   ‘qS rb   rc   r_   ra   r   r   r]   N  s     c                    s   g | ]}ˆ | d  ‘qS )éÿÿÿÿr   ©rB   Úbc)r   r   r   r]   P  s     c                 S   s   g | ]}t  |¡‘qS r   )rR   rO   re   r   r   r   r]   R  s     )r$   rE   ÚzipÚAssertionErrorr%   Z_foreach_negÚtupleZ_foreach_add_Z_foreach_mul_Z_foreach_addcmul_Z_foreach_sub_Z_foreach_neg_Z_foreach_divZ_foreach_reciprocal_Z_foreach_sqrtZ_foreach_mulZ_foreach_div_Z_foreach_addZ_foreach_addcdiv_)r   r4   r5   r6   r7   r,   r-   r   r   r   r
   r   rU   rV   rW   rX   Zexp_avg_sq_sqrtZeps_over_step_sizerY   r   )r,   r-   r   r   rF     sh    ÿ
þ





 
ÿ

rF   )NF)r<   rR   Útypingr   r   r%   r   ZoptimZ	Optimizerr   r=   r'   r1   rG   rF   r   r   r   r   Ú<module>   sd      ùñ1óCó