U
    ¯Ê¹j¨  ã                   @   sB  d Z ddlmZmZ ddlZddlmZ zddlm	Z	 dZ
W n ek
rT   dZ
Y nX zddlmZmZ dZW n ek
r†   dZY nX dad	d
„ Zddd„Zdejeeej eej eejdœdd„Zdejee eej eej eejdœdd„Zdejee eej edœdd„Zdejee eej eejdœdd„ZdS )a   'Fast' Normalization Functions

For GroupNorm and LayerNorm these functions bypass typical AMP upcast to float32.

Additionally, for LayerNorm, the APEX fused LN is used if available (which also does not upcast)

Hacked together by / Copyright 2022 Ross Wightman
é    )ÚListÚOptionalN)Ú
functional)Úfused_layer_norm_affineTF)Úfused_rms_norm_affineÚfused_rms_normc                   C   s   t S ©N©Ú_USE_FAST_NORM© r   r   úT/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/layers/fast_norm.pyÚis_fast_norm   s    r   c                 C   s   | a d S r   r	   )Úenabler   r   r   Úset_fast_norm#   s    r   çñhãˆµøä>)ÚxÚ
num_groupsÚweightÚbiasÚepsÚreturnc              
   C   s”   t j ¡ rt | ||||¡S t  ¡ rZt  ¡ }|  |¡| |¡|d k	rN| |¡nd   } }}t jj	j
dd��  t | ||||¡W  5 Q R £ S Q R X d S ©NF)Zenabled)ÚtorchÚjitÚis_scriptingÚFZ
group_normÚis_autocast_enabledÚget_autocast_gpu_dtypeÚtoÚcudaÚampÚautocast)r   r   r   r   r   Údtr   r   r   Úfast_group_norm(   s    
.r#   )r   Únormalized_shaper   r   r   r   c              
   C   s¨   t j ¡ rt | ||||¡S tr0t| ||||ƒS t  ¡ rnt  ¡ }|  	|¡| 	|¡|d k	rb| 	|¡nd   } }}t j
jjdd��  t | ||||¡W  5 Q R £ S Q R X d S r   )r   r   r   r   Z
layer_normÚhas_apexr   r   r   r   r   r    r!   )r   r$   r   r   r   r"   r   r   r   Úfast_layer_norm>   s    
.r&   ©r   r$   r   r   c                 C   s€   t |ƒ}tj ¡ r4|dkst‚tj| dd� d¡}n&ttd| d dƒƒ}tj| |dd�}| t 	|| ¡ } |d k	r|| | } | S )Né   éÿÿÿÿ)ÚdimT)r*   Zkeepdim)
Úlenr   r   r   ÚAssertionErrorÚvarZ	unsqueezeÚtupleÚrangeZrsqrt)r   r$   r   r   Z	norm_ndimÚvZdimsr   r   r   Úrms_normW   s    
r1   )r   r$   r   r   r   c                 C   sL   t j ¡ rt| |||ƒS tr>|d kr0t| ||ƒS t| |||ƒS t| |||ƒS r   )r   r   r   r1   Úhas_apex_rmsnormr   r   r'   r   r   r   Úfast_rms_normm   s    
r3   )T)NNr   )NNr   )Nr   )Nr   )Ú__doc__Útypingr   r   r   Ztorch.nnr   r   Z#apex.normalization.fused_layer_normr   r%   ÚImportErrorr   r   r2   r
   r   r   ZTensorÚintÚfloatr#   r&   r1   r3   r   r   r   r   Ú<module>   sn   


   ûú   ûú  üü  üû