U
    ¯Ê¹j  ã                   @   sN   d Z ddlZddlZddlZddlZddlmZ e e	¡Z
G dd„ deƒZdS )z† Cosine Scheduler

Cosine LR schedule with warmup, cycle/restarts, noise, k-decay.

Hacked together by / Copyright 2021 Ross Wightman
é    Né   )Ú	Schedulerc                       sJ   e Zd ZdZdejjeeeeeddœ‡ fdd„Z	dd„ Z
ddd„Z‡  ZS )ÚCosineLRSchedulera\  
    Cosine decay with restarts.
    This is described in the paper https://arxiv.org/abs/1608.03983.

    Inspiration from
    https://github.com/allenai/allennlp/blob/master/allennlp/training/learning_rate_schedulers/cosine.py

    k-decay option based on `k-decay: A New Method For Learning Rate Schedule` - https://arxiv.org/abs/2004.05909
    ç        ç      ð?r   r   FTNçq=
×£på?é*   )Ú	optimizerÚ	t_initialÚlr_minÚ	cycle_mulÚcycle_decayÚcycle_limitÚreturnc              
      sÐ   t ƒ j|d|
|||||d� |dks(t‚|dks4t‚|dkrV|dkrV|dkrVt d¡ |ˆ _|ˆ _|ˆ _|ˆ _|ˆ _	|ˆ _
ˆˆ _|	ˆ _|ˆ _ˆ j
rº‡ ‡fdd„ˆ jD ƒˆ _t ƒ  ˆ j¡ ndd„ ˆ jD ƒˆ _d S )	NÚlr)Zparam_group_fieldÚt_in_epochsÚnoise_range_tÚ	noise_pctÚ	noise_stdÚ
noise_seedÚ
initializer   r   zjCosine annealing scheduler will have no effect on the learning rate since t_initial = t_mul = eta_mul = 1.c                    s   g | ]}|ˆ ˆ j  ‘qS © )Úwarmup_t©Ú.0Úv©ÚselfÚwarmup_lr_initr   úW/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/scheduler/cosine_lr.pyÚ
<listcomp>K   s     z.CosineLRScheduler.__init__.<locals>.<listcomp>c                 S   s   g | ]}d ‘qS )r   r   ©r   Ú_r   r   r   r    N   s     )ÚsuperÚ__init__ÚAssertionErrorÚ_loggerÚwarningr
   r   r   r   r   r   r   Úwarmup_prefixÚk_decayÚbase_valuesÚwarmup_stepsZupdate_groups)r   r	   r
   r   r   r   r   r   r   r(   r   r   r   r   r   r)   r   ©Ú	__class__r   r   r$      s:    øÿzCosineLRScheduler.__init__c                    s  ˆˆj k r"‡‡fdd„ˆjD ƒ}nðˆjr2ˆˆj  ‰ˆjdkr˜t t dˆˆj dˆj   ˆj¡¡}ˆj| ˆj ‰ˆdˆj|  dˆj  ˆj  ‰nˆˆj }ˆj‰ˆˆj|  ‰ˆj| ‰ ‡ fdd„ˆj	D ƒ}ˆj
‰|ˆjk rþ‡‡‡‡fdd„|D ƒ}n‡fdd„ˆj	D ƒ}|S )Nc                    s   g | ]}ˆ j ˆ|  ‘qS r   )r   )r   Ús)r   Útr   r   r    R   s     z-CosineLRScheduler._get_lr.<locals>.<listcomp>r   c                    s   g | ]}|ˆ  ‘qS r   r   r   )Úgammar   r   r    a   s     c              
      s@   g | ]8}ˆj d |ˆj   dt tjˆˆ   ˆˆ   ¡   ‘qS )g      à?r   )r   ÚmathÚcosÚpi)r   Zlr_max)Úkr   Út_currÚt_ir   r   r    e   s   ÿc                    s   g | ]
}ˆ j ‘qS r   )r   r!   )r   r   r   r    j   s     )r   r+   r(   r   r1   ÚfloorÚlogr
   r   r*   r)   r   )r   r/   ZlrsÚiZlr_max_valuesr   )r0   r4   r   r/   r5   r6   r   Ú_get_lrP   s(    


($


þzCosineLRScheduler._get_lrc                 C   sR   t d|p| jƒ}| jdkr$| j| S tt | j | j| d  d| j  ¡ƒS d S )Nr   r   )Úmaxr   r   r
   Úintr1   r7   )r   Zcyclesr   r   r   Úget_cycle_lengthn   s    

z"CosineLRScheduler.get_cycle_length)r   r   r   r   r   r   FTNr   r   r   r   T)r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtorchZoptimZ	Optimizerr<   Úfloatr$   r:   r=   Ú__classcell__r   r   r,   r   r      s4                 ïî3r   )rA   Úloggingr1   ÚnumpyÚnprB   Z	schedulerr   Ú	getLoggerr>   r&   r   r   r   r   r   Ú<module>   s   
