U
    •Ê¹j!.  ã                   @   sX   d dl Z d dlmZ d dlmZ G dd„ deƒZG dd„ deƒZG dd„ de jjƒZ	dS )	é    N)ÚFunctionc                   @   s$   e Zd Zedd„ ƒZedd„ ƒZdS )ÚSyncBatchNormc
              
      sr  |j tjd�s$|j tjd�s$| ¡ }|d k	r4| ¡ }t| ¡ | d¡ ƒ}
|
dkrh|	dk rhtd|
› �ƒ‚|j	d }| ¡ dkrÈt 
||¡\}}tjd| ¡ | d¡ |j|jd�}tj|||gdd�‰ ntjd| d |j|jd�‰ | ¡ d	k�rNˆ  ¡ }tjd||	 ˆ jˆ jd�}tj|ˆ |d
d� t ||	|f¡‰ tjˆ |dd�\}}}nL‡ fdd„t|	ƒD ƒ}tj|ˆ |d
d� tj|dd�‰ tjˆ |dd�\}}}tj ¡ �r²tj ¡ �sØ| d¡dk}|| }|| }|| }| d¡}|d k	�r|j|jk�r| |j¡}t ||||||||¡\}}|   ||||| tj!¡¡ || _"| ¡ dk�rdt #||||||¡S t $|¡S d S )N©Zmemory_formaté   é   zEExpected more than 1 value per channel when training, got input size r   )r   ©ÚdtypeÚdevice©ÚdimZglooF©Zasync_opc                    s   g | ]}t  ˆ ¡‘qS © )ÚtorchÚ
empty_like)Ú.0Ú_©Úcombinedr   úZ/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/nn/modules/_functions.pyÚ
<listcomp>B   s    z)SyncBatchNorm.forward.<locals>.<listcomp>éÿÿÿÿ)%Úis_contiguousr   Úchannels_lastÚchannels_last_3dÚ
contiguousÚintÚnumelÚsizeÚ
ValueErrorÚshapeZbatch_norm_statsÚfullr   r	   ÚcatÚzerosZ_get_backend_nameÚemptyÚdistZall_gather_into_tensorZreshapeÚsplitÚrangeZ
all_gatherÚstackÚcudaZis_availableZis_current_stream_capturingZsqueezeÚviewÚtoZ#batch_norm_gather_stats_with_countsÚsave_for_backwardZint32Úprocess_groupZbatch_norm_elemtr   )ÚselfÚinputÚweightZbiasZrunning_meanZrunning_varÚepsZmomentumr,   Z
world_sizer   Únum_channelsÚmeanÚinvstdÚcountZcombined_sizeZcombined_flatZmean_allZ
invstd_allZ	count_allZcombined_listÚmaskÚcountsr   r   r   Úforward   s€    ÿþ
ü
ýý
ÿ	
øzSyncBatchNorm.forwardc                 C   s¤  |j tjd�s$|j tjd�s$| ¡ }| j\}}}}}d  } }}	| j}
| ¡ dk�rFt |||||| j	d | j	d | j	d ¡\}}}}	| j	d �r|j
d }tj||gdd�}tjj|tjjj|
dd� t ||¡\}}|d k	rø|j|jkrø| |j¡}t ||||||||¡}|d k�s&| j	d �s*d }|d k�s@| j	d �sŽd }	nH|j
d }| j	d �rŽtjd| |j|jd�}tjj|tjjj|
dd� |||	d d d d d d f	S )	Nr   r   r   r   r
   Fr   r   )r   r   r   r   r   Úsaved_tensorsr,   r   Zbatch_norm_backward_reduceZneeds_input_gradr   r!   ÚdistributedZ
all_reduceZReduceOpZSUMr%   r   r*   Zbatch_norm_backward_elemtr"   r	   )r-   Úgrad_outputZsaved_inputr/   r2   r3   Zcount_tensorÚ
grad_inputZgrad_weightZ	grad_biasr,   Zsum_dyZ
sum_dy_xmur1   r   r   r   r   Úbackwardp   sv    ÿþø
   ÿø
ý   ÿzSyncBatchNorm.backwardN©Ú__name__Ú
__module__Ú__qualname__Ústaticmethodr7   r<   r   r   r   r   r      s   
gr   c                   @   s&   e Zd Zed	dd„ƒZedd„ ƒZdS )
ÚCrossMapLRN2dç-Cëâ6?ç      è?r   c                 C   sè  || _ || _|| _|| _d | _| ¡ dkr>td| ¡ › d�ƒ‚| jpJ| ¡ | _| ¡ }|  d¡}|  d¡}|  d¡}	|  d¡}
| |¡ | j |¡ |}t	j
|d|d� t| j d d d ƒ}||krÊ|n|}| j dd¡}| ¡  t|ƒD ]}| | d|¡¡ qìtd|ƒD ]Œ}| j d|d ¡}| j d|¡}| |¡ ||| d k �rp| d|| d ¡}|j|dd	� ||k�r| d|| ¡}|j|d
d	� �q| j | j| j  ¡ | j¡ t	j
| j| j |d� | |¡ |  ||¡ |S )Né   z,CrossMapLRN2d: Expected input to be 4D, got z
D instead.r   r   r   é   ©Úout©Úalphar   )r   rJ   ÚbetaÚkÚscaler   r   ÚnewÚ
resize_as_r   Úpowr   ÚselectÚzero_r&   Úadd_Zcopy_Úmul_r+   )Úctxr.   r   rJ   rK   rL   ÚoutputÚ
batch_sizeÚchannelsÚinput_heightÚinput_widthZinput_squareZpre_padZpre_pad_cropZscale_firstÚcZscale_previousZscale_currentZsquare_nextZsquare_previousr   r   r   r7   ¼   sL    







zCrossMapLRN2d.forwardc                 C   sŠ  | j \}}| ¡ }| d¡}| d¡}| d¡}| d¡}| || j d ||¡}	| ||¡}
d| j | j | j }t| j| jd d  ƒ}| |¡ tj| j	| j |d� 
|¡ |	 ¡  |	 d||¡}t|ƒD ]ª}tj|| || |d� | | j	| ¡ tj|	 dd| jd ¡dd|
d� t|ƒD ]R}|
 |	|| j d  ¡ || | j|| | |
| d� |
j|	| d	d
� �q&qÐ|d d d d fS )Nr   r   r   rF   rG   F)ZkeepdimrH   )Úvaluer   rI   )r8   rN   r   rJ   rK   r   rO   r   rP   rM   rT   rR   Znarrowr&   ÚmulZdiv_ÚsumrS   Zaddcmul_)rU   r:   r.   rV   r;   rW   rX   rY   rZ   Zpaddded_ratioZaccum_ratioZcache_ratio_valueZinversePrePadZpadded_ratio_centerÚnr[   r   r   r   r<   õ   s@    




ÿ
ÿ   ÿ"zCrossMapLRN2d.backwardN)rC   rD   r   r=   r   r   r   r   rB   º   s   8rB   c                   @   s$   e Zd Zedd„ ƒZedd„ ƒZdS )ÚBackwardHookFunctionc                 G   s   | j dd„ |D ƒŽ  |S )Nc                 S   s   g | ]}|j s|‘qS r   )Zrequires_grad)r   Úargr   r   r   r     s      z0BackwardHookFunction.forward.<locals>.<listcomp>)Zmark_non_differentiable©rU   Úargsr   r   r   r7     s    zBackwardHookFunction.forwardc                 G   s   |S )Nr   rb   r   r   r   r<     s    zBackwardHookFunction.backwardNr=   r   r   r   r   r`     s   
r`   )
r   Ztorch.distributedr9   r$   Ztorch.autograd.functionr   r   rB   Zautogradr`   r   r   r   r   Ú<module>   s    5^