U
    •Ê¹j%  ã                
   @   sö   d dl Z d dlZd dlmZmZmZmZmZmZm	Z	m
Z
 ddlmZ d dlmZ d dlmZ d dlmZ dd	gZe	ejee eed
f eeef f eej dœdd„Zdee ee eeeeef   eeee	eejf    ee dœdd	„ZdS )é    N)ÚAnyÚDictÚListÚOptionalÚSequenceÚTupleÚUnionÚcasté   )ÚModule©Ú_get_device_index)Úautocast)ÚExceptionWrapperÚ	get_a_varÚparallel_apply.)ÚobjÚreturnc                 C   sx   t | tjƒr| S t | ttfƒrBtt| ƒD ]}t |tjƒr(|  S q(t | tƒrttt|  ¡ ƒD ]}t |tjƒrZ|  S qZd S )N)	Ú
isinstanceÚtorchÚTensorÚlistÚtupleÚmapr   ÚdictÚitems)r   Úresult© r   ú_/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/nn/parallel/parallel_apply.pyr      s    


)ÚmodulesÚinputsÚ
kwargs_tupÚdevicesr   c           
   	      sÒ  t | ƒt |ƒks,tdt | ƒ› dt |ƒ› �ƒ‚|dk	rJt | ƒt |ƒksft‚nttttf i ƒft | ƒ }|dk	r„t | ƒt |ƒks’t‚ndgt | ƒ }dd„ |D ƒ}dd„ |D ƒ}t ¡ ‰i ‰t 	¡ t 
¡  ‰‰dttttttf ttttjf  ttjj ddœ‡‡‡‡fdd	„‰ t | ƒd
k�rj‡ fdd„tt| ||||ƒƒD ƒ}|D ]}| ¡  �qD|D ]}| ¡  �qXn&ˆ d| d |d |d |d |d ƒ g }tt |ƒƒD ],}ˆ| }	t|	tƒ�rÀ|	 ¡  | |	¡ �q |S )a~  Applies each `module` in :attr:`modules` in parallel on arguments
    contained in :attr:`inputs` (positional) and :attr:`kwargs_tup` (keyword)
    on each of :attr:`devices`.

    Args:
        modules (Module): modules to be parallelized
        inputs (tensor): inputs to the modules
        devices (list of int or torch.device): CUDA devices

    :attr:`modules`, :attr:`inputs`, :attr:`kwargs_tup` (if given), and
    :attr:`devices` (if given) should all have same length. Moreover, each
    element of :attr:`inputs` can either be a single object as the only argument
    to a module, or a collection of positional arguments.
    zThe number of modules z& is not equal to the number of inputs Nc                 S   s   g | ]}t |d ƒ‘qS )Tr   ©Ú.0Úxr   r   r   Ú
<listcomp>6   s     z"parallel_apply.<locals>.<listcomp>c                 S   s   g | ]}t j |¡‘qS r   )r   ÚcudaÚcurrent_streamr#   r   r   r   r&   7   s     )ÚiÚmoduleÚinputÚkwargsÚdeviceÚstreamr   c                    s.  t  ˆ¡ |d krTt|ƒ}|d krLˆ� td| › d�d�ˆ| < W 5 Q R X d S | ¡ }|d krht j |¡}z€t j |¡�R t j |¡�: t	ˆ d��$ t
|ttfƒs¦|f}|||Ž}W 5 Q R X W 5 Q R X W 5 Q R X ˆ� |ˆ| < W 5 Q R X W n@ tk
�r(   ˆ�  td| › d|› �d�ˆ| < W 5 Q R X Y nX d S )Nzin replica zQ, no device was provided and no tensor input was found; device cannot be resolved)Úwhere)Zenabledz on device )r   Zset_grad_enabledr   r   Z
get_devicer'   r(   r-   r.   r   r   r   r   Ú	Exception)r)   r*   r+   r,   r-   r.   ÚtÚoutput)Úautocast_enabledÚgrad_enabledÚlockÚresultsr   r   Ú_worker<   s0    

ÿ((ÿzparallel_apply.<locals>._workeré   c              
      s4   g | ],\}\}}}}}t jˆ ||||||fd �‘qS ))ÚtargetÚargs)Ú	threadingÚThread)r$   r)   r*   r+   r,   r-   r.   )r7   r   r   r&   ^   s   þÿr   )NN)ÚlenÚAssertionErrorr	   r   Ústrr   r;   ÚLockr   Zis_grad_enabledZis_autocast_enabledÚintr   r   r   r-   r'   ZStreamÚ	enumerateÚzipÚstartÚjoinÚranger   r   ÚreraiseÚappend)
r   r    r!   r"   ÚstreamsÚthreadsÚthreadÚoutputsr)   r2   r   )r7   r3   r4   r5   r6   r   r      sN    ,  ú

ù!
ý&)NN)r;   r   Útypingr   r   r   r   r   r   r   r	   r   r   Ztorch.cuda._utilsr   Ztorch.cuda.ampr   Ztorch._utilsr   Ú__all__r   r   r?   rA   r-   r   r   r   r   r   Ú<module>   s"   (8  üû