U
    •Ê¹jî(  ã                   @   sT  d dl Z d dl mZ ddlmZmZmZmZmZmZm	Z	 d dl
mZ d dlmZmZ ddgZd	d
„ ZG dd„ deƒZde› de	› de› d�e_dee ee ee ee ee ee ee eeeeeeedœdd„Zee ee ee ee ee ee eeeeeeedœdd„Zee ee ee ee ee ee eeeeeeedœdd„ZdS )é    N)ÚTensoré   )Ú	OptimizerÚ_use_grad_for_differentiableÚ
_get_valueÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_foreach_docÚ_maximize_doc)Úis_compiling)ÚListÚOptionalÚASGDÚasgdc                 C   s   t | tjƒst | ¡S | S ©N©Ú
isinstanceÚtorchr   Útensor©Úx© r   úO/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/optim/asgd.pyÚ
_to_tensor   s    
r   c                       sN   e Zd Zdee eedœ‡ fd	d
„Z‡ fdd„Zdd„ Zeddd„ƒZ	‡  Z
S )r   ç{®Gáz„?ç-Cëâ6?ç      è?ç    €„.Ar   NF)ÚforeachÚmaximizeÚdifferentiablec
              
      sV   d|kst d|› �ƒ‚d|ks,t d|› �ƒ‚t||||||||	d�}
tƒ  ||
¡ d S )Nç        zInvalid learning rate: zInvalid weight_decay value: )ÚlrÚlambdÚalphaÚt0Úweight_decayr   r   r    )Ú
ValueErrorÚdictÚsuperÚ__init__)ÚselfÚparamsr"   r#   r$   r%   r&   r   r   r    Údefaults©Ú	__class__r   r   r*      s    ø
zASGD.__init__c                    s  t ƒ  |¡ | jD ](}| dd ¡ | dd¡ | dd¡ qt| j ¡ ƒ}t|ƒdkoft 	|d d ¡}|sŒ|D ]}t 
t|d ƒ¡|d< qpt|ƒdko¨t 	|d d ¡}|sÊ|D ]}t 
|d ¡|d< q²t|ƒdkoæt 	|d d ¡}|�s|D ]}t 
t|d ƒ¡|d< qòd S )	Nr   r   Fr    r   ÚstepÚetaÚmu)r)   Ú__setstate__Úparam_groupsÚ
setdefaultÚlistÚstateÚvaluesÚlenr   Z	is_tensorr   Úfloat)r+   r7   ÚgroupZstate_valuesZstep_is_tensorÚsZeta_is_tensorZmu_is_tensorr.   r   r   r3   /   s0    

ÿ
ÿ
ÿzASGD.__setstate__c           
      C   sÒ   |d D ]Ä}|j d k	r| |¡ |j jr0tdƒ‚| |j ¡ | j| }	t|	ƒdkr”t d¡|	d< t |d ¡|	d< t d¡|	d	< tj|tj	d
�|	d< | |	d	 ¡ | |	d ¡ | |	d ¡ | |	d ¡ qd S )Nr,   z&ASGD does not support sparse gradientsr   r!   r0   r"   r1   g      ð?r2   )Zmemory_formatÚax)
ÚgradÚappendZ	is_sparseÚRuntimeErrorr7   r9   r   r   Z
zeros_likeZpreserve_format)
r+   r;   Úparams_with_gradÚgradsÚmusÚaxsÚetasÚstate_stepsÚpr7   r   r   r   Ú_init_groupI   s&    


 ÿ
zASGD._init_groupc           
      C   s¨   d}|dk	r&t  ¡ � |ƒ }W 5 Q R X | jD ]v}g }g }g }g }g }g }	|  |||||||	¡ t||||||	|d |d |d |d |d |d |d |d	 d
� q,|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr#   r"   r%   r$   r&   r   r   r    )r#   r"   r%   r$   r&   r   r   r    )r   Zenable_gradr4   rH   r   )
r+   ÚclosureZlossr;   rA   rB   rC   rD   rE   rF   r   r   r   r0   `   s:    

òz	ASGD.step)r   r   r   r   r   NFF)N)Ú__name__Ú
__module__Ú__qualname__r   Úboolr*   r3   rH   r   r0   Ú__classcell__r   r   r.   r   r      s"           ööah  Implements Averaged Stochastic Gradient Descent.

    It has been proposed in `Acceleration of stochastic approximation by
    averaging`_.

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        lambd (float, optional): decay term (default: 1e-4)
        alpha (float, optional): power for eta update (default: 0.75)
        t0 (float, optional): point at which to start averaging (default: 1e6)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        zx

    .. _Acceleration of stochastic approximation by averaging:
        https://dl.acm.org/citation.cfm?id=131098

    F)r,   rB   rD   rC   rE   rF   r   r   r    r#   r"   r%   r$   r&   c	                C   sn   |dkrt | |dd�\}}|r0tj ¡ r0tdƒ‚|rDtj ¡ sDt}nt}|| ||||||	|
|||||d� dS )znFunctional API that performs asgd algorithm computation.

    See :class:`~torch.optim.ASGD` for details.
    NF)Z	use_fusedz6torch.jit.script not supported with foreach optimizers)r#   r"   r%   r$   r&   r   r    )r   r   ZjitZis_scriptingr@   Ú_multi_tensor_asgdÚ_single_tensor_asgd)r,   rB   rD   rC   rE   rF   r   r   r    r#   r"   r%   r$   r&   Ú_Úfuncr   r   r   r   ¢   s,    ó)r,   rB   rD   rC   rE   rF   r#   r"   r%   r$   r&   r   r    c                C   sH  dd„ }t | ƒD �]0\}}|| }|s*|n| }|| }|| }|| }|| }t |¡rxt |¡}t |¡}t |¡}|d7 }t|ƒ}|
dkrž|j||
d�}t|ƒ}| d||  ¡ |j|| d� tƒ sÚ| 	¡ dkrò| | 
|¡ |¡¡ n
| |¡ ||d|| |  |	  ƒ}| |¡ |dtd|| ƒ ƒ}| |¡ qd S )Nc                 S   s   t | tjƒst | ¡S | S r   r   r   r   r   r   r   æ   s    
z'_single_tensor_asgd.<locals>._to_tensorr   r   ©r$   )Ú	enumerater   Ú
is_complexÚview_as_realr   ÚaddZmul_Úadd_r   ÚitemÚsubÚmulÚcopy_Úmax)r,   rB   rD   rC   rE   rF   r#   r"   r%   r$   r&   r   r    r   ÚiÚparamr>   r2   r=   r1   Zstep_tr0   Z	eta_valueÚnew_etaÚnew_mur   r   r   rP   Ö   s4    





rP   c             	   C   sÈ  t | ƒdkrd S |rtdƒ‚t | |||||g¡}| ¡ D �]†\\}}}}}}}|r^t |¡}dd„ }||ƒ}||ƒ}||ƒ}t |d¡ |
dkr¸|r¨tj|||
d� ntj|||
d�}t	|d ƒ}t 
|d||  ¡ tj||| d� tt |ƒƒD ]X}tƒ �s||  ¡ dk�r<||  ||  || ¡ || ¡¡ qö||  || ¡ qött |ƒƒD ]d}t|d|| t	|| ƒ|	    ƒ}||  |¡ tdtdt	|| ƒ| ƒ ƒ}||  |¡ �q\q:d S )Nr   z#_foreach ops don't support autogradc                 S   s   dd„ | D ƒS )Nc                 S   s$   g | ]}t  |¡rt  |¡n|‘qS r   )r   rU   rV   )Ú.0Útr   r   r   Ú
<listcomp>/  s    zE_multi_tensor_asgd.<locals>._view_complex_as_real.<locals>.<listcomp>r   )Ztensor_listr   r   r   Ú_view_complex_as_real.  s    ÿz1_multi_tensor_asgd.<locals>._view_complex_as_realr   rS   )r9   ÚAssertionErrorr   Z"_group_tensors_by_device_and_dtyper8   r   Z_foreach_negZ_foreach_add_Z_foreach_addr   Z_foreach_mul_Úranger   rY   rX   rZ   r[   r\   r   r]   )r,   rB   rD   rC   rE   rF   r#   r"   r%   r$   r&   r   r    Zgrouped_tensorsZgrouped_paramsZgrouped_gradsZgrouped_axsZgrouped_musZgrouped_etasZgrouped_state_stepsrQ   re   r1   r^   r`   ra   r   r   r   rO     sD    ÿ  
(ÿrO   )NFF)r   r   Z	optimizerr   r   r   r   r   r	   r
   Ztorch._utilsr   Útypingr   r   Ú__all__r   r   Ú__doc__rM   r:   r   rP   rO   r   r   r   r   Ú<module>   sz   $zóòñ
    õï5ò=ò