U
    ©Ê¹jMy  ã                   @   s�  U d dl Z d dlZd dlmZ d dlmZmZ d dlmZ d dl	m
Z
mZmZmZmZmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
l m!Z! d dl"m#Z# d dl$m%Z%m&Z&m'Z'm(Z(m)Z) d dl*m+Z+ d dl,m-Z-m.Z. d dl/m0Z0 d dl1m2Z2m3Z3m4Z4m5Z5 da6ee j7 e8d< ej9j:Z:G dd„ deƒZ;eG dd„ dƒƒZ<eG dd„ dƒƒZ=e
e>dœdd„Z?dMej@jAee
df eeeBe
f  eeejCeejDeee' ee' f f  e
dœdd„ZEdd „ ZFejjGe#dd!œd"d#„ZHeejGe
f e
e
d$œd%d&„ZIeee=eJf  eeeeJ ee' f d'œd(d)„ZKejGee
df ed*œd+d,„ZLejGee
df ed*œd-d.„ZMejGee
df eeBe
f eed/œd0d1„ZNejGee
df eeBe
f eed/œd2d3„ZOejGee
df eeBe
f eed/œd4d5„ZPe:jQjReLe:jSjReLe:jTjReMe:jUjReLiZVeej@jAef e8d6< e:jWjReNe:jXjReOe:jXjYeOiZZeej@jAef e8d7< e:j[jRePe:jXjYePe:j\jRePiZ]eej@jAef e8d8< d9dd:œejGeejGe
f e>ee eej^ d;œd<d=„Z_eeejGe
f eej^e
f d>œd?d@„Z`ej^ejGeejGe
f ejGdAœdBdC„Zaej^eejjGejj^f ddDœdEdF„ZbejceejGeejG f dGœdHdI„ZddNej^eejC ee< ee e>eej^eeBe<f f dJœdKdL„ZedS )Oé    N)Ú	dataclass)ÚautoÚEnum)Úpartial)	ÚAnyÚCallableÚcastÚDictÚListÚOptionalÚSequenceÚTupleÚUnion)Ú_get_tracer)ÚOP)Ú
get_logger)Ú
DeviceMeshÚDTensor)Ú_operator_dispatch)ÚOpSchema)Ú_PartialÚDTensorSpecÚ	PlacementÚ	ReplicateÚShard)Úredistribute_local_tensor)Úmake_fxÚ
proxy_slot)ÚTensorMetadata)Útree_flattenÚtree_mapÚtree_map_onlyÚtree_unflattenÚloggerc                   @   s   e Zd Zeƒ Zeƒ ZdS )ÚTrainingPhaseN)Ú__name__Ú
__module__Ú__qualname__r   ZFORWARDZBACKWARD© r(   r(   úa/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/distributed/_spmd/distribute.pyr$   %   s   r$   c                   @   s"   e Zd ZU eed< ee ed< dS )ÚSchemaÚmeshÚ
placementsN)r%   r&   r'   r   Ú__annotations__r
   r   r(   r(   r(   r)   r*   *   s   
r*   c                   @   sP   e Zd ZU dZeed< eed< eed< edœdd„Ze	e
jed dœd	d
„ƒZdS )ÚDSymInta?  
    DSymInt represents a value retrieved by a SymInt op from a DTensor. DSymInt
    helps View and Factory ops to determine the placement and shape of the
    output tensor, as those operators either do not have an input DTensor or
    the input DTensor is insufficient to determine the output tensor's placement.
    Úglobal_valueÚlocal_valuer+   )Úreturnc                 C   s   | j | jkS ©N)r0   r/   )Úselfr(   r(   r)   Úis_shard=   s    zDSymInt.is_shard)ÚnodeÚdtensorr1   c                 C   s¸   d}|j tjkr@tt|jd ƒ}| | |¡| ¡  |¡|jd�S |j tj	krh| | 
¡ | ¡  
¡ |jd�S |j tjkr¤tt|jd ƒ}| | |¡| ¡  |¡|jd�S td|j › �ƒ‚d S )Nr   é   )r/   r0   r+   zDSymInt does not support )ÚtargetÚatenÚsym_sizer   ÚintÚargsÚsizeÚto_localÚdevice_meshÚ	sym_numelZnumelÚ
sym_strideÚstrideÚNotImplementedError)Úclsr5   r6   Údimr(   r(   r)   Ú	from_node@   s,    ý
ýýzDSymInt.from_nodeN)r%   r&   r'   Ú__doc__r;   r-   r   Úboolr4   ÚclassmethodÚfxÚNoder   rF   r(   r(   r(   r)   r.   0   s   
r.   )Úobjr1   c                 C   s4   t | tƒsdS d}| jD ]}t |tƒrd} q0q|S )zDcheck if object is 1) DTensor and  2) with any placement of _PartialFT)Ú
isinstancer   r,   r   )rL   Z
is_partialZ	placementr(   r(   r)   Ú_is_partial_dtensor[   s    


rN   .)ÚopÚ
local_argsÚkwargsÚspecsr1   c                    s<   |d kri }ˆ d kri ‰ t t dœ‡ fdd„}| t||ƒ|ŽS )N©Úargr1   c              	      sz   ˆ |  \}}}}t || j| j|  ¡ tj| ji d�}t|t|ƒ|d�}t|t|ƒ|d�}t	| tj
ƒrv| ˆ krvt| ||ƒS | S )N)ÚdtypeÚrequires_gradrB   Zmemory_formatÚis_quantizedZqparams)Útensor_meta)r   rU   rV   rB   ÚtorchZcontiguous_formatrW   r   ÚtuplerM   ÚTensorr   )rT   Ztensor_shaper+   Zcurrent_placementZtarget_placementrX   Zcurrent_specZtarget_spec©rR   r(   r)   Úredistributey   s4    ù	  ÿ  ÿ
ÿÿýz2_dispatch_with_local_tensors.<locals>.redistribute)r   r    )rO   rP   rQ   rR   r]   r(   r\   r)   Ú_dispatch_with_local_tensorsi   s    r^   c                 C   s~   t | ƒ\}}t |jƒ\}}i }t|ƒD ]D\}}	t|	tƒr&|r`|	 ¡ || j|	j|| jf||	j< |	j||< q&t	||ƒ}
||
fS r2   )
r   Úargs_schemaÚ	enumeraterM   r   r=   r+   r,   Ú_local_tensorr"   )r<   Útarget_schemar]   Úflatten_argsÚargs_tree_specZflatten_args_schemaÚ_rR   ÚirT   Úunflattened_argsr(   r(   r)   Ú_update_specs_for_redistribute˜   s    
ø	
ü

rh   )r5   Ú	op_schemar1   c                 C   s®   t | jƒ\}}t |jƒ\}}tttjjf tdœdd„}t	|ƒt	|ƒksLt
‚tt||ƒƒD ]&\}\}}	||ƒrZt|	tƒrZ|	||< qZt||ƒ}
t|
ƒD ]\}}|  ||¡ q”d S )NrS   c                 S   s.   t | tjjƒr$| jtjtjtjfkS t | t	ƒS r2   )
rM   rY   rJ   rK   r8   r9   r:   r@   rA   r;   )rT   r(   r(   r)   Úis_sym_int_or_int¼   s    ýz6_update_node_from_op_schema.<locals>.is_sym_int_or_int)r   r<   r_   r   r;   rY   rJ   rK   rH   ÚlenÚAssertionErrorr`   ÚziprM   r"   Z
update_arg)r5   ri   Ú	flat_argsrd   Zflat_args_schemare   rj   rf   rT   Z
arg_schemar<   Úidxr(   r(   r)   Ú_update_node_from_op_schema¸   s    	

rp   )Únode_to_objrT   r1   c                 C   s>   t |tjjƒr6| | }tƒ r2ttttf |jƒt	= |S |S d S r2   )
rM   rY   rJ   rK   r   r   r	   r   Ú__dict__r   )rq   rT   rL   r(   r(   r)   Ú
_remap_argÐ   s    rs   )Úsizesr+   r1   c                 C   sZ   dd„ | D ƒ}dd„ t | ƒD ƒp&tƒ g}t|ƒ|jksRtdt|ƒ› d|j› d�ƒ‚||fS )Nc                 S   s    g | ]}t |tƒr|jn|‘qS r(   ©rM   r.   r0   ©Ú.0Úsr(   r(   r)   Ú
<listcomp>ß   s    z)unpack_sizes_and_dims.<locals>.<listcomp>c                 S   s*   g | ]"\}}t |tƒr| ¡ rt|ƒ‘qS r(   )rM   r.   r4   r   )rw   rf   Úar(   r(   r)   ry   â   s   
 þz"The number of sharded dimensions (z2) must match number of dimensions in device mesh (z).)r`   r   rk   Úndimrl   )rt   r+   Úlocal_sizesr,   r(   r(   r)   Úunpack_sizes_and_dimsÜ   s    ÿþüÿr}   )r5   r<   r1   c                 C   s¾   t |ƒdks"td| j› d|› �ƒ‚t|d tƒsBtd|d › �ƒ‚t|d tƒsbtd|d › �ƒ‚t|d |d jƒ\}}| jd |f| _t	t
jj| jƒ}tj||d j|ƒ|d j|dd	�S )
Né   zExpect two args but got op z with args r   z*Expect 1st argument to be DTensor but got r7   ú$Expect 2nd argument as list but got F©Zlocal_tensorr?   r,   Ú	run_check)rk   rl   r8   rM   r   Úlistr}   r?   r<   r   rY   Ú_opsÚ
OpOverloadÚ
from_localra   )r5   r<   r|   r,   rO   r(   r(   r)   Úbinop_sym_int_consumer_ruleð   s"    " ÿþ ür†   c           
   	   C   s\   |\}}}}}}dd„ |D ƒ}t j||j|jd�}	tjt  |	| ¡ ||||¡|j|j	dd�S )Nc                 S   s    g | ]}t |tƒr|jn|‘qS r(   ru   rv   r(   r(   r)   ry     s    z7slice_backwad_sym_int_consumer_rule.<locals>.<listcomp>)ÚdevicerU   Fr€   )
rY   Úzerosr‡   rU   r   r…   Zslice_scatterr>   r?   r,   )
r5   r<   Zgrad_outputZinput_sizesrE   ÚstartÚendÚstepr|   Zinput_tensorr(   r(   r)   Ú#slice_backwad_sym_int_consumer_rule  s,    ÿ  ÿ     ÿúrŒ   )r5   r<   rQ   Údefault_meshr1   c                 C   s¦   t |ƒd }tdd„ |D ƒƒr6td| j› d|› d�ƒ‚t|d tƒsVtd|d › �ƒ‚t|d |ƒ\}}|f|dd … ˜| _tt	j
j| jƒ}tj|| j|Ž||d	d
�S )Nr   c                 s   s   | ]}t |tƒV  qd S r2   )rM   r   ©rw   rz   r(   r(   r)   Ú	<genexpr>#  s     z*factory_with_sizes_rule.<locals>.<genexpr>z4Not expect DTensor argument for factory op, but got z with arguments Ú.r   r7   Fr€   )r   Úanyrl   r8   rM   r‚   r}   r<   r   rY   rƒ   r„   r   r…   )r5   r<   rQ   r�   rn   r|   r,   rO   r(   r(   r)   Úfactory_with_sizes_rule  s    ÿ 
ür’   c                 C   s>   t dd„ |ƒ| _ttjj| jƒ}tj|| j|Ž|t	ƒ gdd�S )Nc                 S   s   t | tƒr| jS | S r2   ru   ©rz   r(   r(   r)   Ú<lambda>:  ó    z%factory_arange_rule.<locals>.<lambda>Fr€   )
r    r<   r   rY   rƒ   r„   r8   r   r…   r   ©r5   r<   rQ   r�   rO   r(   r(   r)   Úfactory_arange_rule4  s    
ür—   c                 C   s>   || | _ | _ttjj| jƒ}tj|| j | jŽ|t	ƒ gdd�S )NFr€   )
r<   rQ   r   rY   rƒ   r„   r8   r   r…   r   r–   r(   r(   r)   Údefault_factory_op_ruleD  s    ür˜   ÚVIEW_SYM_INT_CONSUMERSÚFACTORY_SYM_INT_CONSUMERSÚFACTORY_OPSF)Úforce_make_fxr�   )r5   rq   rœ   r�   r1   c             
   C   s  t  ¡ ��  ttt|ƒ| jƒ}ttt|ƒ| jƒ}tt jj	| j
ƒ}tdd„ t|ƒd D ƒƒrö|tkrœt|ƒdks|td|› �ƒ‚t| | |ƒ|| < W 5 Q R £ d S |tkrØ|d k	s´tdƒ‚t| | |||ƒ|| < W 5 Q R £ d S tttjƒsèt‚t d||¡ | j
tjjk�rtjj}tdd„ |ƒ}td	d„ |ƒ}|tk�rXt| | |||ƒ|| < W 5 Q R £ d S t|||tjƒ\}}}	||| < |	d k	�s„t‚|	j�s¤t| |ƒ W 5 Q R £ d S |	j d k	�s´t‚|	j d }
t!||
|	jƒ\}}tt"|||d
�}t#|dd�|ƒ}|j$ %¡  |W  5 Q R £ S Q R X d S )Nc                 s   s    | ]}t |tƒr| ¡ V  qd S r2   )rM   r.   r4   rŽ   r(   r(   r)   r�   |  s     
 z._get_dtensor_dispatch_graph.<locals>.<genexpr>r   zExpect empty kwargs, but got z%Requires default mesh for factory opszYAssuming using local_value from SymInt for %sis mathematically correct. Full args are %s.c                 S   s   t | tƒr| jS | S r2   ru   r“   r(   r(   r)   r”   ™  r•   z-_get_dtensor_dispatch_graph.<locals>.<lambda>c                 S   s   t | tƒr| jS | S r2   ru   r“   r(   r(   r)   r”   ›  r•   )rQ   rR   F)Z_allow_non_fake_inputs)&rY   Zno_gradr    r   rs   r<   rQ   r   rƒ   r„   r8   r‘   r   r™   rk   rl   rš   rM   r#   ÚloggingÚLoggerÚwarningr9   ÚviewÚdefaultZreshaper›   r   r   Z_propagatorZneeds_redistributerp   Zschema_suggestionsrh   r^   r   ÚgraphÚeliminate_dead_code)r5   rq   rœ   r�   r<   rQ   Zop_overloadÚoutri   Zoutput_shardingrb   Zupdated_args_specrg   ÚdispatchÚgmr(   r(   r)   Ú_get_dtensor_dispatch_graphn  s„       ÿü ÿ
   ÿü


  ÿü
r§   )Údtrq   r1   c           	      C   sÖ   t jt jt jdœdd„}| j}t  | j¡}t|ƒ||ƒ}dd„ |jjD ƒ}dd„ |jjD ƒ}t|ƒdkslt‚t|ƒdks|t‚| ||d	 < t	j
|| jtƒ gd
d�||d < t|d	 |dd�}|dk	sÆt‚|||d	  fS )a   
    Creates a graph for a dummy add function from a partial DTensor.
    This dummy add is used for triggering all_reduce on a Partial DTensor
    during the DTensor expansion of the traced graph.
    Also returns the actual DTensor after resharding.
    )ÚgradÚzeror1   c                 S   s   | | S r2   r(   )r©   rª   r(   r(   r)   Ú	dummy_addä  s    z)_build_dummy_add_graph.<locals>.dummy_addc                 S   s   g | ]}|j tjkr|‘qS r(   )rO   r   ÚPLACEHOLDER©rw   Únr(   r(   r)   ry   ì  s      z*_build_dummy_add_graph.<locals>.<listcomp>c                 S   s   g | ]}|j tjkr|‘qS r(   )rO   r   ÚCALL_FUNCTIONr­   r(   r(   r)   ry   í  s      r~   r7   r   F©r�   T)rœ   N)rY   r[   ra   Z
zeros_liker   r¢   Únodesrk   rl   r   r…   r?   r   r§   )	r¨   rq   r«   r©   rª   Z
traced_addZplaceholdersZcall_functionsÚtraced_dispatchr(   r(   r)   Ú_build_dummy_add_graphÚ  s,    
   ÿ  ÿr³   )r¦   r5   rq   r1   c                    s  g }d}|j d D �]Æ}t|tjƒs0| |¡ q|| }t|ƒsL| |¡ qd}tt|ƒ}t||ƒ\}}	dd„ |j	j
D ƒ}
dd„ |j	j
D ƒ}t|
ƒdkr¤t|ƒdks¨t‚|d  |
d ¡ |j	 ¡  |	||
d < i ‰ |j	j
D ]ü}|jtjkrö|ˆ |< qÜ|jtjk�r‚t|j ƒdk�r(t|j d ƒdk�sDtd|j › d	t|j ƒ› �ƒ‚| ˆ |j d d  ¡ ||j d d  |ˆ |j d d  < qÜ|jtjk�r¦t| |jt||jƒƒ | j	 |¡�  | j	 |‡ fd
d„¡ˆ |< W 5 Q R X qÜq|�rú| j	 |¡ | j	 |¡S |S d S )NFr   Tc                 S   s$   g | ]}|j d ks|j dkr|‘qS )Z	wait_commZwait_tensor©Únamer­   r(   r(   r)   ry     s   
 
þz#_convert_output.<locals>.<listcomp>c                 S   s   g | ]}|j d kr|‘qS )Úaddr´   r­   r(   r(   r)   ry     s     
 r7   ú!Expecting single output, but got ú c                    s   ˆ |  S r2   r(   ©r®   ©Zvalue_remapr(   r)   r”   A  r•   z!_convert_output.<locals>.<lambda>)r<   rM   rJ   rK   ÚappendrN   r   r   r³   r¢   r±   rk   rl   Úreplace_all_uses_withr£   rO   r   r¬   ÚOUTPUTZGET_ATTRÚsetattrr8   ÚgetattrÚinserting_beforeÚ	node_copyÚ
erase_nodeÚoutput)r¦   r5   rq   Únew_argsZhas_partialÚargumentrL   r¨   r²   Z
result_objÚwaitr¶   Údtnr(   rº   r)   Ú_convert_outputÿ  s\    


þ

ÿÿþ&
ý(rÈ   )r¦   Únode_replacementsr1   c              
      s  | j jD �]ö}||krq|| }t|jƒ\}}di  }‰ |j jD ]$}|jtjkr@|| ˆ |< |d7 }q@| j  |¡��† |j jD �]h}|jtjkr’q~|jtjk�r¤t	|jƒdksÎt
d|j› dt	|jd ƒ› �ƒ‚|jd }t	|ƒdkrî|d }	n¢d }
t|ƒD ]‚\}}|d k�rqú|jdk�st
‚|jjdk�s0t
‚|jjdk�sBt
‚|
d k�s`|
|jd k�s`t
‚|jd }
|jd |ksút
‚qú|
d k	�sŒt
‚|
}	ˆ |	 }| |¡ q~| j  |‡ fdd	„¡ˆ |< td
d„ ||fD ƒƒr~| ˆ | ¡  �qêq~| j  |¡ W 5 Q R X q| j  ¡  |  ¡  d S )Nr   r7   r·   r¸   Zcall_functionÚ	_operatorÚgetitemc                    s   ˆ |  S r2   r(   r¹   rº   r(   r)   r”   „  r•   z _rebuild_graph.<locals>.<lambda>c                 s   s.   | ]&}t |jtjjƒo$|jjj d ¡V  qdS ))zaten::_foreachzaten::_fused_adamN)rM   r8   rY   rƒ   r„   Z_schemarµ   Ú
startswithr­   r(   r(   r)   r�   …  s
   ü
ÿz!_rebuild_graph.<locals>.<genexpr>)r¢   r±   r   r<   rO   r   r¬   rÀ   r½   rk   rl   r`   r8   r&   r%   r¼   rÁ   ÚallrÂ   r£   Z	recompile)r¦   rÉ   r5   r²   rc   re   rf   rÇ   ÚoutputsrÃ   Úsourcer¤   Únew_noder(   rº   r)   Ú_rebuild_graphI  sZ    

ÿþ



û
rÑ   )r¢   r1   c                    sl   i ‰i ‰t jt jd dœ‡‡fdd„‰ t| jƒD ]8‰t j ˆj‡ ‡fdd„¡ t j ˆj‡ ‡fdd„¡ q.ˆS )N)Úarg_nodeÚconsumerr1   c                    s&   | ˆkr"|ˆ| < ˆ   |g ¡ | ¡ d S r2   )Ú
setdefaultr»   )rÒ   rÓ   )Úlast_consumer_to_nodesÚnode_to_last_consumerr(   r)   Ú_register_final_consumer©  s    z=_get_last_consumer_to_nodes.<locals>._register_final_consumerc                    s
   ˆ | ˆƒS r2   r(   )rÒ   ©r×   r5   r(   r)   r”   °  r•   z-_get_last_consumer_to_nodes.<locals>.<lambda>c                    s
   ˆ | ˆƒS r2   r(   )Z
kwarg_noderØ   r(   r)   r”   ´  r•   )rJ   rK   Úreversedr±   r5   Zmap_argr<   rQ   )r¢   r(   )r×   rÕ   r5   rÖ   r)   Ú_get_last_consumer_to_nodesŸ  s     ÿþrÚ   )r¦   ÚinpsÚschemasr�   Ú_allow_partialr1   c                    s  t dƒadd„ tjD ƒ}i }i }t| jƒ}i }	t| jjƒD �]È\}
}tdk	sPt‚t 	d|
|j
|j¡ |j
tjkrÈ|
t|ƒk sštd|
d › dt|ƒ› d	�ƒ‚tj||
  ¡ ||
 j||
 jd
d�||< �nt|jtjjƒ�rtt||jd  ƒ}t ||¡||< �nät|jtjjƒ�r8t|||d�}|dk	�ræ|||< �n®|j
tjk�r¤|�sXt| ||ƒ}|jd D ]<}t|t j!ƒ�rb|| }t|tƒ�rbt"|j#|jƒ|	|j$< �qb�nB|j
tj%k�rÖt&t't(|ƒ|jƒ}t&t't(|ƒ|j)ƒ}t*t+dd„ |t,| -¡ ƒ ƒƒ‰ |j|k�r¤tˆ ƒdk�r¤t.‡ fdd„ˆ D ƒƒ�s.tdƒ‚t/tdd„ |ƒ}t/tdd„ |ƒ}t/tdd„ |ƒ}t/tdd„ |ƒ}||_||_)t|j||Ž|j||Žˆ d jd�||< n0tˆ ƒdk�sÄtd|j› d�ƒ‚|j||Ž||< nt0d|j
› �ƒ‚||kr:|| D ]}||= �qöq:t1| |ƒ | |	fS )zc
    Returns:
        - transformed graph module
        - map from output name to DTensorSpec
    Zspmd_expc                 S   s   h | ]}t t|ƒ’qS r(   )r¿   Úoperator)rw   rµ   r(   r(   r)   Ú	<setcomp>È  s     z*_convert_to_distributed.<locals>.<setcomp>Nznode%s: op=%s target=%szgot more placeholder nodes (r7   z) than inputs (ú)Fr°   r   )r�   c                 S   s
   t | tƒS r2   )rM   r.   r“   r(   r(   r)   r”     r•   z)_convert_to_distributed.<locals>.<lambda>c                 3   s   | ]}ˆ d  j |j kV  qdS )r   N)r+   )rw   Úd©Zdsymintsr(   r)   r�     s    z*_convert_to_distributed.<locals>.<genexpr>z&all DSymInts must have the same mesh. c                 S   s   | j S r2   ©r0   r“   r(   r(   r)   r”   	  r•   c                 S   s   | j S r2   rã   r“   r(   r(   r)   r”   
  r•   c                 S   s   | j S r2   ©r/   r“   r(   r(   r)   r”     r•   c                 S   s   | j S r2   rä   r“   r(   r(   r)   r”     r•   )r0   r/   r+   zBSPMD expansion does not support SymInt in non-operator nodes, got r�   zUnrecognized node.op type )2r   r#   rÞ   Ú__all__rÚ   r¢   r`   r±   rl   ÚinforO   r8   r   r¬   rk   r   r…   Úcloner+   r,   rM   rY   rƒ   ZOpOverloadPacketr   r<   r.   rF   r„   r§   r½   rÈ   rJ   rK   r*   r?   rµ   r¯   r    r   rs   rQ   r‚   ÚfilterrZ   ÚvaluesrÍ   r!   Ú
ValueErrorrÑ   )r¦   rÛ   rÜ   r�   rÝ   Ú	operatorsrq   rÉ   rÕ   Zoutput_schemasrf   r5   r6   ÚreplacementZinp_argrL   r<   rQ   rP   Zlocal_kwargsZglobal_argsZglobal_kwargsrÒ   r(   râ   r)   Ú_convert_to_distributedº  s˜    
ÿþ
û  ÿ
 ÿÿÿþ

ýÿ
rí   )NN)NF)fr�   rÞ   Údataclassesr   Úenumr   r   Ú	functoolsr   Útypingr   r   r   r	   r
   r   r   r   r   rY   Z(torch.distributed._spmd.experimental_opsZtorch.fxrJ   Z#torch.distributed._spmd.comm_tensorr   Z#torch.distributed._spmd.graph_utilsr   Z!torch.distributed._spmd.log_utilsr   Ztorch.distributed._tensorr   r   Z"torch.distributed._tensor.dispatchr   Z#torch.distributed._tensor.op_schemar   Z)torch.distributed._tensor.placement_typesr   r   r   r   r   Z&torch.distributed._tensor.redistributer   Z"torch.fx.experimental.proxy_tensorr   r   Ztorch.fx.passes.shape_propr   Ztorch.utils._pytreer   r    r!   r"   r#   rž   r-   Zopsr9   r$   r*   r.   rH   rN   rƒ   r„   Ústrr[   ÚSizer^   rh   rK   rp   rs   r;   r}   r†   rŒ   r’   r—   r˜   Z_unsafe_viewr¡   ÚexpandZslice_backwardr    r™   ÚfullZaranger‰   rš   Zscalar_tensorrˆ   r›   ZGraphModuler§   r³   rÈ   rÑ   ZGraphrÚ   rí   r(   r(   r(   r)   Ú<module>   s   ,*  ÷
ÿÿÿö/  þ 
þ

û

û

û    ü   ý
   ýûúm þ&üKýWþ  ûú