U
    ¯Ê¹jV#  ã                   @   s&  d dl mZmZ d dlZd dlmZ d dlmZ zd dlmZmZ dZ	W n e
k
r`   dZ	Y nX d dlmZmZ d	d
gZG dd	„ d	eƒZdee ee eee  eee eeeeeedœdd
„Zee ee eee  eeeeeeedœ
dd„Zee ee eee  eeeeeeedœ
dd„ZdS )é    )Úupdate_wrapperÚwrapsN)ÚTensor)Ú	Optimizer)Ú_use_grad_for_differentiableÚ_default_to_fused_or_foreachTF)ÚListÚOptionalÚSGDWÚsgdwc                       s\   e Zd Zdddddœeee edœ‡ fdd„Z‡ fdd	„Zd
d„ Ze 	¡ ddd„ƒZ
‡  ZS )r
   çü©ñÒMbP?r   FN)ÚmaximizeÚforeachÚdifferentiablec             
      sˆ   |dk rt d|› �ƒ‚|dk r,t d|› �ƒ‚|dk rBt d|› �ƒ‚t||||||||	d�}
|rv|dksn|dkrvt dƒ‚tƒ  ||
¡ d S )Ng        zInvalid learning rate: zInvalid momentum value: zInvalid weight_decay value: )ÚlrÚmomentumÚ	dampeningÚweight_decayÚnesterovr   r   r   r   z8Nesterov momentum requires a momentum and zero dampening)Ú
ValueErrorÚdictÚsuperÚ__init__)ÚselfÚparamsr   r   r   r   r   r   r   r   Údefaults©Ú	__class__© úN/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/optim/sgdw.pyr      s&        üzSGDW.__init__c                    sL   t ƒ  |¡ | jD ]4}| dd¡ | dd¡ | dd ¡ | dd¡ qd S )Nr   Fr   r   r   )r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr   r   r   r    .   s    
zSGDW.__setstate__c                 C   sn   d}|d D ]\}|j d k	r| |¡ | |j ¡ |j jr<d}| j| }d|krZ| d ¡ q| |d ¡ q|S )NFr   TÚmomentum_buffer)ÚgradÚappendÚ	is_sparser#   )r   r$   Úparams_with_gradÚd_p_listÚmomentum_buffer_listÚhas_sparse_gradÚpr#   r   r   r   Ú_init_group6   s    


zSGDW._init_groupc                 C   s²   d}|dk	r&t  ¡ � |ƒ }W 5 Q R X | jD ]€}g }g }g }|  ||||¡}t||||d |d |d |d |d |d ||d d	� t||ƒD ]\}}	| j| }
|	|
d
< q�q,|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r   )r   r   r   r   r   r   r,   r   r%   )ÚtorchZenable_gradr!   r.   r   Úzipr#   )r   ÚclosureZlossr$   r)   r*   r+   r,   r-   r%   r#   r   r   r   ÚstepJ   s4    

õ
z	SGDW.step)r   r   r   r   F)N)Ú__name__Ú
__module__Ú__qualname__Úboolr	   r   r    r.   r/   Zno_gradr2   Ú__classcell__r   r   r   r   r
      s$        ù	õ	õ)r   r*   r+   r,   r   r   r   r   r   r   r   c                C   sŒ   t rPttdƒrP|dkr8tj ¡ s4t| ddd�\}}nd}|rTtj ¡ rTtdƒ‚nd}|rhtj ¡ sht}nt	}|| |||||||	||
d�
 dS )zlFunctional API that performs SGD algorithm computation.

    See :class:`~torch.optim.SGD` for details.
    Ú"_group_tensors_by_device_and_dtypeNF)r   Z	use_fusedz6torch.jit.script not supported with foreach optimizers)r   r   r   r   r   r,   r   )
Úhas_recent_ptÚhasattrr   r/   ZjitZis_scriptingr   ÚRuntimeErrorÚ_multi_tensor_sgdwÚ_single_tensor_sgdw)r   r*   r+   r,   r   r   r   r   r   r   r   Ú_Úfuncr   r   r   r   t   s.    

ö)
r   r*   r+   r   r   r   r   r   r   r,   c                C   s®   t | ƒD ] \}
}|s||
 n||
  }| d||  ¡ |dkr˜||
 }|d krht |¡ ¡ }|||
< n| |¡j|d| d� |r”|j||d�}n|}|j|| d� qd S )Nç      ð?r   é   ©Úalpha)Ú	enumerateÚmul_r/   ÚcloneÚdetachÚadd_Úadd)r   r*   r+   r   r   r   r   r   r   r,   ÚiÚparamZd_pÚbufr   r   r   r=   ©   s    
r=   )
r   Úgradsr+   r   r   r   r   r   r   r,   c                C   sÄ  t | ƒdkrd S tj| ||gdd�}
|
 ¡ D �]�\\}}}}|	oPtdd„ |D ƒƒ}|r`t |¡}t | d||  ¡ |dk�rxg }d}tt |ƒƒD ](}|| d kr¬d} q¼q’| 	|| ¡ q’|rât ||¡ tj
||d| d	� nzg }tt |ƒƒD ]h}|| d k�r,t || ¡ ¡  } ||< ||| < n$|| }| |¡j|| d| d	� | 	|¡ qò|�rttj
|||d	� n|}|�s’tj
||| d	� q,tt |ƒƒD ]}|| j|| | d	� �qžq,d S )
Nr   T)Zwith_indicesc                 s   s   | ]}|j V  qd S )N)r(   )Ú.0r&   r   r   r   Ú	<genexpr>ß   s     z%_multi_tensor_sgdw.<locals>.<genexpr>r@   FrA   rB   )Úlenr   r8   ÚvaluesÚanyr/   Z_foreach_negZ_foreach_mul_Úranger'   Z_foreach_add_rF   rG   rE   rH   )r   rM   r+   r   r   r   r   r   r   r,   Zgrouped_tensorsZdevice_paramsZdevice_gradsZdevice_momentum_buffer_listÚindicesZdevice_has_sparse_gradZbufsZall_states_with_momentum_bufferrJ   rL   r   r   r   r<   Ì   sJ     ÿ

ÿr<   )NN)Ú	functoolsr   r   r/   r   Ztorch.optim.optimizerr   r   r   r9   ÚImportErrorÚtypingr   r	   Ú__all__r
   r6   Úfloatr   r=   r<   r   r   r   r   Ú<module>   s`   
j  ù
ò6
õ$
õ