U
    ¯Ê¹j  ã                   @   s®   d Z ddlZddlZddlmZ ddlZddlmZ ddlm	Z	 e 
e¡Zdd„ Zdd
d„Zdd„ Zdd„ Zddd„Zdd„ Zdd„ Zdd„ Zdeee ee dœdd„ZdS )zZ Distributed training/validation utils

Hacked together by / Copyright 2020 Ross Wightman
é    N)ÚOptional)Údistributedé   )Úunwrap_modelc                 C   s&   |   ¡ }tj|tjjd� || }|S )N©Úop)ÚcloneÚdistÚ
all_reduceÚReduceOpÚSUM)ZtensorÚnÚrt© r   úU/var/www/html/bwave/python/venv/lib/python3.8/site-packages/timm/utils/distributed.pyÚreduce_tensor   s    r   Fc                 C   sb   t | ƒjdd�D ]L\}}d|ks(d|kr|rNtjj|tjjd� |t|ƒ }qtj 	|d¡ qd S )NT)ZrecurseZrunning_meanZrunning_varr   r   )
r   Znamed_buffersÚtorchr   r
   r	   r   r   ÚfloatÚ	broadcast)ÚmodelÚ
world_sizeÚreduceZbn_nameZbn_bufr   r   r   Údistribute_bn   s    r   c                 C   s
   | j dkS ©Nr   )Úrank©Úargsr   r   r   Úis_global_primary%   s    r   c                 C   s
   | j dkS r   )Ú
local_rankr   r   r   r   Úis_local_primary)   s    r   c                 C   s   |rt | ƒS t| ƒS )N)r   r   )r   Úlocalr   r   r   Ú
is_primary-   s    r!   c                   C   s<   dt jkrtt jd ƒdkS dt jkr8tt jd ƒdkS dS )NÚ
WORLD_SIZEr   ÚSLURM_NTASKSF©ÚosÚenvironÚintr   r   r   r   Úis_distributed_env1   s
    

r(   c                  C   sˆ   d} dD ] }|t jkrtt j| ƒ}  q*qd}dD ] }|t jkr2tt j| ƒ} qTq2d}dD ] }|t jkr\tt j| ƒ} q~q\| ||fS )Nr   )Ú
LOCAL_RANKZMPI_LOCALRANKIDZSLURM_LOCALIDZOMPI_COMM_WORLD_LOCAL_RANK)ÚRANKZPMI_RANKÚSLURM_PROCIDZOMPI_COMM_WORLD_RANKr   )r"   ZPMI_SIZEr#   ZOMPI_COMM_WORLD_SIZEr$   )r   ÚvÚglobal_rankr   r   r   r   Úworld_info_from_env9   s     


r.   c                 C   s€   d| _ d| _d| _d| _tt| ddƒt| dd ƒt| dd ƒd�}|d | _|d	 | _|d
 | _|d | _|d | _ t | j¡}|S )NFr   r   ÚdeviceÚcudaÚdist_backendÚdist_url©r/   r1   r2   r   r-   r   r   )r   r   r   r   Úinit_distributed_device_soÚgetattrr/   r   )r   Úresultr/   r   r   r   Úinit_distributed_deviceO   s     


ý




r7   r0   r3   c           	      C   s^  d}d}d}d}|d kr(d| kr$dnd}|p.d}t ƒ rÀdtjkrŒtƒ \}}}t|ƒtjd	< t|ƒtjd
< t|ƒtjd< tjj||||d� n0tƒ \}}}tjj||d� tj ¡ }tj 	¡ }d}d| krâtj
 ¡ sâtd| › d�ƒ‚|�r4| dk�r4| jddd�^} }|�r&t d|d › d| › d�¡ | › d|› �} |  d¡�rLtj
 | ¡ t| ||||d�S )NFr   r   r0   ZncclZgloozenv://r+   r)   r*   r"   )ÚbackendÚinit_methodr   r   )r8   r9   TzCUDA is not available but z was specified.Úcpuú:)Úmaxsplitzdevice index z removed from specified (z).zcuda:)r/   r-   r   r   r   )r(   r%   r&   r.   Ústrr   r   Zinit_process_groupZget_world_sizeZget_rankr0   Zis_availableÚAssertionErrorÚsplitÚ_loggerÚwarningÚ
startswithZ
set_deviceÚdict)	r/   r1   r2   r   r   r-   r   Ú_Z
device_idxr   r   r   r4   d   sV    
üþ

ûr4   )F)F)r0   NN)Ú__doc__Úloggingr%   Útypingr   r   r   r	   r   r   Ú	getLoggerÚ__name__r@   r   r   r   r   r!   r(   r.   r7   r=   r4   r   r   r   r   Ú<module>   s.   


   ýý