U
    ¯Ê¹jX  ã                   @   s$   d Z ddlZd
dd„Zddd	„ZdS )aP   Adaptive Gradient Clipping

An impl of AGC, as per (https://arxiv.org/abs/2102.06171):

@article{brock2021high,
  author={Andrew Brock and Soham De and Samuel L. Smith and Karen Simonyan},
  title={High-Performance Large-Scale Image Recognition Without Normalization},
  journal={arXiv preprint arXiv:},
  year={2021}
}

Code references:
  * Official JAX impl (paper authors): https://github.com/deepmind/deepmind-research/tree/master/nfnets
  * Phil Wang's PyTorch gist: https://gist.github.com/lucidrains/0d6560077edac419ab5d3aa29e674d5c

Hacked together by / Copyright 2021 Ross Wightman
é    Nç       @c                 C   s4   | j dkr|  |¡S | j|ttd| j ƒƒdd�S d S )Né   T)ÚdimZkeepdim)ÚndimZnormÚtupleÚrange)ÚxÚ	norm_type© r
   úM/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/utils/agc.pyÚunitwise_norm   s    

r   ç{®Gáz„?çü©ñÒMbP?c                 C   sš   t | tjƒr| g} | D ]~}|jd kr&q| ¡ }|j ¡ }t||d�j|d� |¡}t||d�}|||jdd�  }	t 	||k ||	¡}
|j ¡  
|
¡ qd S )N)r	   )Úming�íµ ÷Æ°>)Ú
isinstanceÚtorchZTensorZgradÚdetachr   Zclamp_Zmul_ÚclampÚwhereZcopy_)Ú
parametersZclip_factorÚepsr	   ÚpZp_dataZg_dataZmax_normZ	grad_normZclipped_gradZ	new_gradsr
   r
   r   Úadaptive_clip_grad   s    

r   )r   )r   r   r   )Ú__doc__r   r   r   r
   r
   r
   r   Ú<module>   s   
	