U
    ©Ê¹jy  ã                   @   s®   d dl Z d dlmZmZmZmZmZmZmZ d dl	Z	d dl
mZ d dlmZ ddlmZmZ dZeƒ dejeee	jj  ejdœdd	„ƒZejed
œdd„ZG dd„ dƒZdS )é    N)ÚAnyÚDictÚIterableÚListÚOptionalÚSetÚTuple)ÚDistributedDataParallelé   )Ú_get_registryÚcontractÚ )ÚmoduleÚignored_modulesÚreturnc                 K   sV   t j d¡ d|kr>t|d tt jfƒs>tdt|d ƒ› �ƒ‚t|d�j	| f|Ž | S )zðReplicates a module

    Args:
        module (torch.nn.Module): module to replicate

    Example::
        >>> # xdoctest: +REQUIRES(module:torch._C._distributed_c10d)
        >>> module = nn.Linear(3, 3)
        >>> replicate(module)
    ztorch.distributed.replicateÚ	device_idz6Expected device_id to be int or torch.device, but got )r   )
ÚtorchZ_CZ_log_api_usage_onceÚ
isinstanceÚintÚdeviceÚRuntimeErrorÚtypeÚ_ReplicateStateÚmark_module)r   r   Úkwargs© r   úf/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/distributed/_composable/replicate.pyÚ	replicate   s    ÿr   ©r   r   c                 C   s   dt | ƒkS )z+Check if module is marked with fully_shard.Zfully_shard)r   )r   r   r   r   Ú_is_fully_sharded(   s    r   c                   @   s¦   e Zd Zeeejj  ddœdd„Zejddœdd„Z	e
fejeddœd	d
„Zddœdd„Zejeedf eeef edœdd„Zejeej ejejdœdd„ZdS )r   N)r   r   c                 C   sP   d | _ d| _t ¡ | _i | _|d k	r,t|ƒntƒ | _dd„ | jD ƒ| _g | _	d S )NFc                 S   s   h | ]}|  ¡ D ]}|’qqS r   )Ú
parameters)Ú.0ÚmÚpr   r   r   Ú	<setcomp>6   s    
  z+_ReplicateState.__init__.<locals>.<setcomp>)
r   Úhas_initializedÚnnZParameterListÚ_param_listr   Úsetr   Úignored_paramsÚ_param_names)Úselfr   r   r   r   Ú__init__.   s    
ÿÿz_ReplicateState.__init__r   c                 K   sH   t |ƒrtdƒ‚|| _dt |¡_|j| jdd� | | j	¡ || _
d S )NzGCannot apply `replicate()` on a Module already managed by `fully_shard`FT)Zwith_kwargs)r   ÚAssertionErrorr   r   ÚstateZ_params_collectedZregister_forward_pre_hookÚforward_pre_hookZregister_forward_hookÚforward_post_hookr   )r+   r   r   r   r   r   r   <   s    ÿz_ReplicateState.mark_module)r   Úprefixr   c                 C   sœ   t |ƒrd S || jkrd S |tkr,|› d�nt}|jdd�D ]2\}}|| jkr<| j |¡ | j |› |› �¡ q<| ¡ D ]\}}| j	||› |› �d� qxd S )NÚ.F)Zrecurse)r   r1   )
r   r   Ú_ROOT_MODULE_PREFIXZnamed_parametersr)   r'   Úappendr*   Znamed_childrenÚ_collect_params)r+   r   r1   Zrecurse_prefixÚnr#   ÚnameZchild_moduler   r   r   r5   H   s    
ÿ
z_ReplicateState._collect_params)r   c                 C   sÀ   | j r
d S d| _ |  | j¡ | jt | j¡_d| jkr”| jd d k	r~| jd }t|t	j
ƒrp|jdkrpd | jd< qˆ|g| jd< n
d | jd< | j d¡ t| jf| jŽ| _t | j¡t | j¡_d S )NTr   ÚcpuZ
device_ids)r%   r5   r   r*   r   r.   Z_replicate_param_namesr   r   r   r   r   Úpopr	   r'   Ú_ddpÚweakrefÚrefZ_ddp_weakref)r+   r   r   r   r   Úinit_helper^   s    


z_ReplicateState.init_helper.)r   Úargsr   r   c                 C   s"   |   ¡  | jj||Ž\}}||fS ©N)r=   r:   Z_pre_forward)r+   r   r>   r   r   r   r   r/   }   s    z _ReplicateState.forward_pre_hook)r   ÚinputÚoutputr   c                 C   s   | j  |¡S r?   )r:   Z_post_forward)r+   r   r@   rA   r   r   r   r0   „   s    z!_ReplicateState.forward_post_hook)Ú__name__Ú
__module__Ú__qualname__r   r   r   r&   ÚModuler,   r   r3   Ústrr5   r=   r   r   r   r/   ZTensorr0   r   r   r   r   r   -   s&   ÿ þ  
 
þ	ûr   )N)r;   Útypingr   r   r   r   r   r   r   r   Ztorch.nnr&   Ztorch.nn.parallelr	   r   r   r3   rE   r   Úboolr   r   r   r   r   r   Ú<module>   s   $ þü