U
    ¯Ê¹j»  ã                   @   sÆ   d Z ddlmZ ddlZddlmZ G dd„ deƒZdeej eej eej eee	e	e	e	dœ	d	d
„Z
eej eej eej e	e	e	e	edœdd„Zeej eej eej e	e	e	e	edœdd„ZdS )z« Lion Optimizer
Paper: `Symbolic Discovery of Optimization Algorithms` - https://arxiv.org/abs/2302.06675
Original Impl: https://github.com/google/automl/tree/master/lion
é    )ÚListN)Ú	Optimizerc                       s@   e Zd ZdZd‡ fdd„	Z‡ fd	d
„Ze ¡ ddd„ƒZ‡  Z	S )ÚLionzImplements Lion algorithm.ç-Cëâ6?©gÍÌÌÌÌÌì?g®Gáz®ï?ç        FNc                    s’   d|kst d |¡ƒ‚d|d   kr.dk sBn t d |d ¡ƒ‚d|d   krZdk snn t d |d ¡ƒ‚t|||||d�}tƒ  ||¡ d	S )
aÔ  Initialize the hyperparameters.

        Args:
          params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
          lr (float, optional): learning rate (default: 1e-4)
          betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.99))
          weight_decay (float, optional): weight decay coefficient (default: 0)
        r   zInvalid learning rate: {}r   g      ð?z%Invalid beta parameter at index 0: {}é   z%Invalid beta parameter at index 1: {})ÚlrÚbetasÚweight_decayÚforeachÚmaximizeN)Ú
ValueErrorÚformatÚdictÚsuperÚ__init__)ÚselfÚparamsr	   r
   r   r   r   Údefaults©Ú	__class__© úN/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/optim/lion.pyr      s    ûzLion.__init__c                    s4   t ƒ  |¡ | jD ]}| dd¡ | dd ¡ qd S )Nr   Fr   )r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr   r   r   r   ?   s    
zLion.__setstate__c                 C   sð   d}|dk	r&t  ¡ � |ƒ }W 5 Q R X | jD ]¾}g }g }g }|d \}}|d D ]n}	|	jdkr`qP| |	¡ |	jjrztdƒ‚| |	j¡ | j|	 }
t|
ƒdkr°t j	|	t j
d�|
d< | |
d ¡ qPt||||||d |d	 |d
 |d d�	 q,|S )zÑPerforms a single optimization step.

        Args:
          closure (callable, optional): A closure that reevaluates the model
            and returns the loss.

        Returns:
          the loss.
        Nr
   r   z&Lion does not support sparse gradientsr   )Zmemory_formatÚexp_avgr	   r   r   r   )Úbeta1Úbeta2r	   r   r   r   )ÚtorchZenable_gradr   ÚgradÚappendZ	is_sparseÚRuntimeErrorr   ÚlenZ
zeros_likeZpreserve_formatÚlion)r   ÚclosureZlossr   Zparams_with_gradÚgradsÚexp_avgsr    r!   Úpr   r   r   r   ÚstepE   s@    




÷z	Lion.step)r   r   r   FN)N)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r"   Zno_gradr,   Ú__classcell__r   r   r   r   r      s        ù#r   F)	r   r)   r*   r   r   r    r!   r	   r   c          
   
   C   sV   |dkrd}|r"t j ¡ r"tdƒ‚|r6t j ¡ s6t}	nt}	|	| |||||||d� dS )z=Functional API that performs Lion algorithm computation.
    NFz6torch.jit.script not supported with foreach optimizers)r    r!   r	   r   r   )r"   ZjitZis_scriptingr%   Ú_multi_tensor_lionÚ_single_tensor_lion)
r   r)   r*   r   r   r    r!   r	   r   Úfuncr   r   r   r'   z   s"    ør'   )r   r)   r*   r    r!   r	   r   r   c                C   s¬   t | ƒD ]ž\}}	|s|| n||  }
|| }t |	¡rVt |
¡}
t |¡}t |	¡}	|	 d||  ¡ | |¡j|
d| d�}|	jt |¡| d� | |
d| ¡ qd S )Nr   ©Úalpha)	Ú	enumerater"   Ú
is_complexÚview_as_realZmul_ÚmulZadd_ÚsignZlerp_)r   r)   r*   r    r!   r	   r   r   ÚiÚparamr#   r   Úupdater   r   r   r3   ¢   s    



r3   c          	      C   sÄ   t | ƒdkrd S |r"t t|ƒ¡}dd„ |D ƒ}dd„ |D ƒ}dd„ | D ƒ} t | d||  ¡ t ||¡}tj||d| d� dd„ |D ƒ}tj| || d� t ||¡ tj||d| d� d S )	Nr   c                 S   s$   g | ]}t  |¡rt  |¡n|‘qS r   ©r"   r8   r9   ©Ú.0Úxr   r   r   Ú
<listcomp>Ò   s     z&_multi_tensor_lion.<locals>.<listcomp>c                 S   s$   g | ]}t  |¡rt  |¡n|‘qS r   r?   r@   r   r   r   rC   Ó   s     c                 S   s$   g | ]}t  |¡rt  |¡n|‘qS r   r?   r@   r   r   r   rC   Ô   s     r   r5   c                 S   s   g | ]}|  ¡ ‘qS r   )r;   )rA   Úur   r   r   rC   Ý   s     )r&   r"   Z_foreach_negÚtupleZ_foreach_mul_Z_foreach_mulZ_foreach_add_)	r   r)   r*   r    r!   r	   r   r   Zupdatesr   r   r   r2   Á   s    r2   )FN)r0   Útypingr   r"   Ztorch.optim.optimizerr   r   ZTensorÚboolÚfloatr'   r3   r2   r   r   r   r   Ú<module>   sF   g  ùô)÷ ÷