U
    ©Ê¹jR  ã                   @   s�  U d dl mZmZ d dlmZmZ d dlmZ d dlmZ d dl	m
Z
mZ d dlmZmZmZmZmZmZmZmZmZ d dlmZ d dlZd dlmZ d dlZd dlmZ d dlm   m!Z" d d	lm#Z# d d
l$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+m,Z,m-Z- d dl.m/Z/ d dl0m1Z1m2Z2m3Z3 d dl4m5Z5 d dl6m7Z7 G dd„ deƒZ8G dd„ de1ƒZ9ej#j:ej#j:dœdd„Z;e,ƒ a<ej=ee/ dœdd„Z>eej?j@eeAef eeAejBf dœdd„ƒZCejDjEZEedd „ ƒZFd<d"d#„ZGd$d%„ ZHd&d'„ ZId=d(d)„ZJd>d*d+„ZKd!d!d!d!d!d,d,ddd-œ	d.d/„ZLeEjMjeGeEjMjNe
eJeEjOjNƒeEjPjNe
eKeEjQjNƒeEjRjNe
eKeEjSjNƒeEjTje
eIeEjUjƒeEjVjNe
eJeEjWjNƒeEjTjNe
eJeEjUjNƒeEjXjYe
eHeEjZjYƒeEj[jYe
eHeEj\jYƒeEj]jYe
eHeEj^jYƒeEj_jNe
eJeEj`jNƒeEjajYeLeEj%jYe%iZbejDjcjdjYejDjcjejYhZfeejgjh eid0< e#j:e#j:dœd1d2„ZjeG d3d4„ d4ƒƒZkeeee8  e-eeekd5œd6d7„Zld8Zmd?eee8  eee#j:ge#j:f  ee- d9œd:d;„ZndS )@é    )ÚABCÚabstractmethod)ÚcontextmanagerÚnullcontext)Úcopy)Ú	dataclass)ÚpartialÚwraps)	ÚAnyÚCallableÚcastÚDictÚListÚOptionalÚSetÚTupleÚUnion)Úmake_fxN)Úfx)Únative_layer_norm_backward)ÚFakeTensorMode)Úgradients_tagging)ÚDataParallelÚDTensorExpandModeÚParallelMode)Ú	Placement)Ú_PyTreeCodeGenÚ_PyTreeInfoÚCodeGen)Ú	stateless)ÚNamedMemberAccessorc                   @   sN   e Zd ZdZeeejjejjdœdd„ƒZ	ee
jeej e
jdœdd„ƒZdS )	ÚOverrideaÕ  
    Override the tracing and transformation behavior of :meth:`~torch.distributed._spmd.compile`.
    This is useful when any part of the model is not traceable or if you prefer
    to not trace it due to any reason. More specifically, users can implement
    :meth:`torch.distributed._spmd.Override.replacement` to replace an original
    submodule with the return new submodule. The new submodule contains
    operations that users preferred to be traced, which simply be a dummy
    placeholder operator. After tracing, users can implement
    :meth:`torch.distributed._spmd.Override.transform` to transform the traced
    graph, where the dummy placeholder operator serves as an anchor to insert
    new sub-graphs.
    )ÚfqnÚorig_submoduleÚreturnc                 C   s   dS )aê  
        Implement this method to return a new :class:`nn.Module` instance to
        replace the ``orig_submodule`` argument in the model. This helps if
        ``orig_submodule`` is not traceable or should not be traced.

        Args:
            fqn (str): fully quantified name of the submodule.
            orig_submodule (class:`nn.Module`): original submodule instance to replace.

        Returns:
            A new :class:`nn.Module` instance to replace the original one.
        N© )Úselfr"   r#   r%   r%   úZ/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/distributed/_spmd/api.pyÚreplacement0   s    zOverride.replacement)ÚgmÚ
flat_stater$   c                 C   s   dS )a.  
        Given a DTensor-expanded graph and sharding schema for every node,
        conduct additional transformation for the sub-graph from the :class:`nn.Module`
        returned by :meth:`torch.distributed._spmd.Override.replacement` if
        necessary.

        Args:
            gm (:class:`fx.Graph`): a DTensor-expanded graph.
            flat_state (List[str, :class:`Tensor`]): a reference to the list of
                flattened state. The elements in ``flat_state`` map to the first
                ``len(flat_state)`` placeholders in the graph. The transformation
                can add state to or remove state from ``flat_state`` as long as
                it keeps ``flat_state`` and the placeholders consistent.

        Returns:
            The :class:`fx.Graph` after transformation.
        Nr%   )r&   r)   r*   r%   r%   r'   Ú	transform@   s    zOverride.transformN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚstrÚtorchÚnnÚModuler(   r   ÚGraphModuler   ÚTensorr+   r%   r%   r%   r'   r!   "   s   ür!   c                   @   s$   e Zd Zeedœdd„Zdd„ ZdS )Ú_PyTreeCodeGenOutputsOnly)Úargsr$   c                 G   s   |S ©Nr%   )r&   r7   r%   r%   r'   Úprocess_inputs\   s    z(_PyTreeCodeGenOutputsOnly.process_inputsc                 C   s   t  | ||¡S r8   )r   Ú
gen_fn_def)r&   Z	free_varsZmaybe_return_annotationr%   r%   r'   r:   `   s    z$_PyTreeCodeGenOutputsOnly.gen_fn_defN)r,   r-   r.   r
   r9   r:   r%   r%   r%   r'   r6   Z   s   r6   )r)   r$   c                 C   s,   t tdd| jjjjd�d�| j_|  ¡  | S )züMove the responsibility of flattening the input arguments from the
    graph module to the caller.

    Example:

        output = gm(my_struct)

        gm = gm(to_caller_flattened_graph_module)

        output = gm(*pytree.flatten(my_struct)[0])
    N)Ú	orig_argsZin_specÚout_spec)Úpytree_info)r6   r   Z_graphZ_codegenr=   r<   Ú	recompile)r)   r%   r%   r'   Ú!_to_caller_flattened_graph_moduled   s    
úÿ

r?   ©ÚtZ
placementsc                 C   s   |t jt| ƒ< d S r8   )Údtensor_expand_modeZ_placements_overrideÚidr@   r%   r%   r'   Ú_override_placements„   s    rD   )ÚoptÚnamed_statesÚparamsc              	   c   sp   | d k	st ‚t| jƒ}|D ]}|| | j|| < q| jd }|d }| ¡ |d< z
d V  W 5 ||d< || _X d S )Nr   rG   )ÚAssertionErrorr   ÚstateZparam_groupsÚvalues)rE   rF   rG   Zorig_statesÚnZparam_groupZorig_paramsr%   r%   r'   Ú_rematerialize_optimizer‰   s    


rL   c                  c   s:   dd„ } t jjj}| jt jj_z
d V  W 5 |t jj_X d S )Nc                   S   s   dS )NTr%   r%   r%   r%   r'   Úf_true«   s    z_enable_compile.<locals>.f_true)r1   Z_utilsZis_compilingÚ__code__)rM   Zorig_is_compiling_coder%   r%   r'   Ú_enable_compile¦   s    

rO   é   c                 C   s4   t jj| ||d�}t| |ƒD ]\}}| |¡ qd S )N)Úalpha)ÚatenÚ_foreach_addr   ÚzipÚcopy_)r&   ÚotherrQ   Úself_updatedÚsÚs_ur%   r%   r'   Ú_foreach_add_decomp¶   s    rZ   c                 C   s*   | |ƒ}t ||ƒD ]\}}| |¡ qd S r8   ©rT   rU   )Úopr&   rW   rX   rY   r%   r%   r'   Ú_foreach_unaop_decomp¼   s    r]   c                 C   s,   | ||ƒ}t ||ƒD ]\}}| |¡ qd S r8   r[   )r\   r&   rV   rW   rX   rY   r%   r%   r'   Ú_foreach_binop_list_decompÂ   s    
r^   c                 C   s,   | ||ƒ}t ||ƒD ]\}}| |¡ qd S r8   r[   )r\   r&   ÚscalarrW   rX   rY   r%   r%   r'   Ú_foreach_binop_scalar_decompÈ   s    
r`   c                 C   s0   | ||||ƒ}t ||ƒD ]\}}| |¡ qd S r8   r[   )r\   r&   Ztensor1Ztensor2r_   rW   rX   rY   r%   r%   r'   Ú_foreach_addcop_scalar_decompÎ   s    ra   T©	ÚlrÚbeta1Úbeta2Úweight_decayÚepsÚamsgradÚmaximizeÚ
grad_scaleÚ	found_infc       	         C   s€   | ||||f}t jj| |||||||||	|
||||d�}tt||ƒƒD ]4\}\}}|dkr\qFt||ƒD ]\}}| |¡ qfqFd S )Nrb   rP   )rR   Z_fused_adamÚdefaultÚ	enumeraterT   rU   )r&   ZgradsZexp_avgsZexp_avg_sqsZmax_exp_avg_sqsZstate_stepsrc   rd   re   rf   rg   rh   ri   rj   rk   Z
orig_tupleZupdated_tupleÚidxÚorigÚupdatedÚoÚur%   r%   r'   Ú_fused_adam_decompÔ   s.    ñrs   ÚDEDUP_TARGETSc                 C   sx   i }| j jD ]^}t |j¡\}}|jtkr|jf|˜}| |d ¡}|d krT|||< q| |¡ | j  	|¡ q|  
¡  | S r8   )ÚgraphZnodesÚpytreeÚtree_flattenr7   Útargetrt   ÚgetZreplace_all_uses_withZ
erase_noder>   )r)   Zargs_to_nodeÚnoder7   Ú_Zargs_keyZunique_noder%   r%   r'   Ú_dedup_collectives,  s    


r|   c                   @   s@   e Zd ZU ejed< ejed< ee	j
j ed< ee	j ed< dS )Ú_CompiledResultr)   ÚmodrE   r*   N)r,   r-   r.   r   r4   Ú__annotations__r2   r3   r   r1   ÚoptimÚ	Optimizerr   r5   r%   r%   r%   r'   r}   C  s   


r}   )ÚfuncÚmodule_overrideÚparallel_moder7   Úkwargsr$   c              
      s<  d\‰‰t  t|ƒt| ¡ ƒ ¡d D ]F}t|tjƒrJˆd ksFtdƒ‚|‰t|tj	j
ƒr&ˆd kshtdƒ‚|‰q&ˆd k	s~tdƒ‚ˆr²tˆƒ‰ ttjjd dœ‡ ‡‡fdd„‰ˆd	ˆƒ tˆjd
d�ƒ}tˆjd
d�ƒ}i }ˆd k	�r
| ¡ D ] \}	}
|
ˆjkrèˆj|
 ||	< qèt|tƒ‰‡‡‡fdd„}ˆ�r.dnd}ˆ�r‚tƒ ‰tt|ƒ‰tjtjdœ‡‡fdd„}t  tj||¡}t  tj||¡}tƒ �B tjjd
d��( tt|| ƒ|td
d�|||||ƒ}W 5 Q R X W 5 Q R X ||–}| |ˆˆ||||¡}t  ||g¡\}}t|ƒ}t|ƒ}ˆ�r.ˆD ]}|  ||¡}�qt!|ˆˆ|ƒS )N)NNr   z%Only support single nn.Module for nowz%Only support single Optimizer for nowz5Couldn't find nn.Module instances from the arguments.)Ú
fqn_prefixÚmoduler$   c                    sz   ˆD ]p}|  ¡ D ]b\}}t|ƒdkr&q| dkr:| d | n|}| ||¡}t|ƒt|ƒkrfˆ||ƒ qˆ  ||¡ qqd S )Nr   Ú Ú.)Znamed_childrenÚlenr(   rC   Zswap_submodule)r†   r‡   ÚoverrideÚnameÚchildr"   Ú	new_child)Úaccessorrƒ   Úswapr%   r'   r�   e  s    z_compile.<locals>.swaprˆ   F)Zremove_duplicatec                    s˜   t  ˆ||–¡�~ ˆr"tˆ||ƒntƒ �\ ˆ r6t|ƒntƒ � | ||Ž}W 5 Q R X |tˆ ¡ ƒt| ¡ ƒfW  5 Q R £ W  5 Q R £ S Q R X W 5 Q R X d S r8   )r   Z_reparametrize_modulerL   r   r   ÚlistÚ
parametersrJ   )r‚   rG   ÚbuffersrF   r7   r…   Úret)Úis_data_parallel_moder~   rE   r%   r'   Ústateless_func†  s     ÿþ  ÿüz _compile.<locals>.stateless_funcZfakeZsymbolic)Úargr$   c                    s6   ˆ  | ¡}dg| j }|ˆ j  t ¡ 9  < | |¡S )NrP   )Zfrom_tensorÚndimZinput_batch_dimÚdistZget_world_sizeÚrepeat)r—   Zfake_argZarg_dims)Údata_parallel_modeÚ	fake_moder%   r'   Ú_get_full_batch_arg�  s    
z%_compile.<locals>._get_full_batch_arg)Z	check_nan)Útracing_modeZdecomposition_tableZ_allow_non_fake_inputs)"rv   rw   r‘   rJ   Ú
isinstancer2   r3   rH   r1   r€   r�   r    r0   ÚdictZnamed_parametersZnamed_buffersÚitemsrI   r   r   r   r5   Ztree_map_onlyrO   ZautogradZdetect_anomalyr   r   ÚSPMD_DECOMP_TABLEÚ	partitionr?   r|   r+   r}   )r‚   rƒ   r„   r7   r…   r—   rG   r“   rF   rK   Úpr–   rž   r�   r)   Zparams_and_buffersr*   r{   r‹   r%   )r�   r›   rœ   r•   r~   rƒ   rE   r�   r'   Ú_compileK  sŠ    "




ýýü    ûþùr¥   Z_compiled_obj)rƒ   Úgm_transformationr„   c                    s   t dœ‡ ‡‡fdd„}|S )a¢  
    Compile and optimize a callable, which can be a train step within a training
    loop. This method will extract :class:`nn.Module` and :class:`torch.optim.Optimizer`
    instances from the input arguments and trace operations applied to their
    parameters and states.

    Args:
        module_override (Optional[List[Override]]): a list of Override instances
            that will be applied to the module in order. The :class:`Override`
            objects provide :class:`nn.Module` replacements during tracing and a
            graph transformation function after tracing. (Default: ``None``)
        gm_transformation (Optional[Callable[fx.GraphModule, fx.GraphModule]]):
            a callback that will be called after the original callable is
            compiled and distributed (usually after the first iteration) to
            transform the compiled GraphModule into a new optimized one.
        parallel_mode (Optional[ParallelMode]): a :class:`ParallelMode` object
            that specifies how to parallelize the callable. Each ParallelMode
            would have its own strategy to partition the model and the captured
            graph (Default: ``None``)
    ©r‚   c                    s    t ˆ ƒ‡ ‡‡‡‡fdd„ƒ‰ˆS )Nc            	         s   |r|  dd¡nd}d}ˆj td ¡}|d kr`d}ˆd kr>tnˆ}tˆ ˆ|f| ž|Ž}|ˆjt< |jt | |g¡d  }t	 
¡ �– |r–ˆr–ˆ|jƒ|_|sª|j|Ž d }n\z|j|d|iŽd }W nB tk
�r } z"dt|ƒkræ|‚|j|Ž d }W 5 d }~X Y nX |W  5 Q R £ S Q R X d S )NÚlast_train_stepFTr   Z	last_iter)ÚpopÚ__dict__ry   ÚCOMPILED_OBJECT_KEYrB   r¥   r*   rv   rw   r1   Zno_gradr)   Ú	TypeErrorr0   )	r7   r…   r¨   Z
first_iterZcompiled_objÚmodeZ	flat_inpsÚoutputÚe)r‚   r¦   rƒ   r„   Úwrapperr%   r'   r°     s0    ÿ

ÿ z'compile.<locals>.inner.<locals>.wrapper)r	   r§   ©r¦   rƒ   r„   )r‚   r°   r'   Úinner  s    *zcompile.<locals>.inner)r   )rƒ   r¦   r„   r²   r%   r±   r'   Úcompileò  s    .r³   )rP   )rP   )rP   )NNN)oÚabcr   r   Ú
contextlibr   r   r   Údataclassesr   Ú	functoolsr   r	   Útypingr
   r   r   r   r   r   r   r   r   Z	functorchr   r1   Ztorch.distributedÚdistributedr™   Z)torch.distributed._functional_collectivesZtorch.nnr2   Ztorch.utils._pytreeÚutilsZ_pytreerv   r   Ztorch._decomp.decompositionsr   Ztorch._subclasses.fake_tensorr   Z%torch.distributed._spmd.data_parallelr   Z%torch.distributed._spmd.parallel_moder   r   r   Ztorch.distributed._tensorr   Ztorch.fx.graphr   r   r   Ztorch.nn.utilsr   Z%torch.nn.utils._named_member_accessorr    r!   r6   r4   r?   rB   r5   rD   r€   r�   r0   Ú	ParameterrL   ZopsrR   rO   rZ   r]   r^   r`   ra   rs   Z_foreach_add_ZScalarrS   Z_foreach_addcdiv_Z_foreach_addcdivZ_foreach_addcmul_Z_foreach_addcmulZ_foreach_div_Z_foreach_divZ_foreach_mul_Z_foreach_mulZ_foreach_neg_rl   Z_foreach_negZ_foreach_reciprocal_Z_foreach_reciprocalZ_foreach_sqrt_Z_foreach_sqrtZ_foreach_sub_Z_foreach_subZ_fused_adam_r¢   Zc10d_functionalZ
all_reduceZwait_tensorrt   Z_opsZ
OpOverloadr   r|   r}   r¥   r«   r³   r%   r%   r%   r'   Ú<module>   s   ,8

ý



ð.   ÿ  ÿ  ÿ  ÿ  ÿ  ÿ  ÿ  ÿ  ÿ  ÿ  ß&

þ
ú %   ý
ý