U
    ªÊ¹jM›  ã                   @   s  d dl mZmZ d dlmZmZmZmZmZm	Z	 d dl
Z
d dlmZ d dlZd dlZd dlm  mZ d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZ ddlmZmZ dd	l m!Z! d dl"Z"e!j#Z$d
d„ Z%dd„ Z&dd„ Z'dd„ Z(G dd„ dƒZ)e)ƒ Z*dd„ Z+dd„ Z,dd„ Z-dd„ Z.dd„ Z/ej0dœdd „Z1ej0dœd!d"„Z2ej0eej0ej0f d#œd$d%„Z3d&d'„ Z4d(d)„ Z5ej6e7d*œd+d,„Z8d-d.„ Z9e" :d¡d/d0„ ƒZ;d1d2„ Z<d3d4„ Z=d5d6„ Z>d7d8„ Z?d9d:„ Z@dFej0eej0ej0f d#œd<d=„ZAdGe
jj0eBeBd@œdAdB„ZCdHdDdE„ZDdS )Ié    )Úis_sym_nodeÚpy_sym_types)Úhint_intÚmagic_methodsÚmethod_to_operatorÚfree_symbolsÚis_symbol_binding_fx_nodeÚfind_symbol_binding_fx_nodesN©Údefaultdict)Úgraph_drawer)ÚTupleé   )Úfx_graph_cseÚget_aten_target)Úconfigc                 C   s   | j  dd¡S )NÚ	recomputeF)ÚmetaÚget©Únode© r   ú\/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/_functorch/partitioners.pyÚmust_recompute   s    r   c                 C   s$   d}| j jD ]}t|ƒr dS qdS )NFT)ÚgraphÚnodesr   )Úfx_gÚfoundr   r   r   r   Úhas_recomputable_ops   s
    r   c                 C   s<   | j jD ].}t|ƒrt|jdƒrtjj|jjkr dS qdS )NÚtagsTF)	r   r   r   ÚhasattrÚtargetÚtorchÚTagÚnondeterministic_seededr   )r   r   r   r   r   Úhas_recomputable_rng_ops#   s    $r%   c                 C   s6   t | jd tjtjfƒrdS t | jd tjƒs2t‚dS )NÚvalr   é   )Ú
isinstancer   r"   ÚSymIntZSymBoolZSymFloatÚAssertionErrorr   r   r   r   Úsym_node_size)   s    r+   c                   @   s   e Zd Zdd„ ZdS )ÚInvalidNodeBasec                 C   s   dS )NzInvalid Noder   )Úselfr   r   r   Ú__repr__0   s    zInvalidNodeBase.__repr__N)Ú__name__Ú
__module__Ú__qualname__r.   r   r   r   r   r,   /   s   r,   c           	         sx  t  ¡ }i ‰ |D ] }| |j¡}|j|_|ˆ |< q| jD ]°}||krHq8q8|jdkr\tˆ |< q8|jdkrºt 	|j
|jf¡d }‡ fdd„|D ƒ}t|ƒr tˆ |< q8| |‡ fdd„¡ˆ |< q8|jdkrÞ| |‡ fd	d„¡ˆ |< q8|jd
kr8q8g }|D ]f}t|t jƒ�rN|ˆ k�rtd|› d�ƒ‚tˆ | tƒ�r>td|› d�ƒ‚| ˆ | ¡ qò| |¡ qò| |¡ | ¡  | ¡  |S )a‘  
    Given a graph, extracts out a subgraph that takes the specified nodes as
    inputs and returns the specified outputs.

    This includes specifying non-placeholder nodes as inputs.

    The general strategy is to initialize all inputs with proxies as we
    encounter them, and trace through the graph, only keeping values which take
    in valid proxies. Then, all dead code is eliminated.
    ÚplaceholderÚcall_functionr   c                    s&   g | ]}t |tjƒrt ˆ | tƒ‘qS r   )r(   ÚfxÚNoder,   )Ú.0Úx©Úenvr   r   Ú
<listcomp>S   s      z6_extract_graph_with_inputs_outputs.<locals>.<listcomp>c                    s   ˆ |  S ©Nr   ©r7   r8   r   r   Ú<lambda>W   ó    z4_extract_graph_with_inputs_outputs.<locals>.<lambda>Zget_attrc                    s   ˆ |  S r;   r   r<   r8   r   r   r=   Y   r>   ÚoutputzNode z couldn't be found in envz was invalid, but is output)r4   ÚGraphr2   Únamer   r   ÚopÚInvalidNodeÚpytreeÚtree_flattenÚargsÚkwargsÚanyÚ	node_copyr(   r5   ÚRuntimeErrorr,   r*   Úappendr?   Úeliminate_dead_codeZlint)	Zjoint_graphÚinputsÚoutputsÚ	new_graphr   Únew_nodeZall_argsZoutput_valuesr7   r   r8   r   Ú"_extract_graph_with_inputs_outputs7   sD    







 
rQ   c                 C   s(   | j dko&d| jko&t| ƒ o&t| ƒ S ©Nr2   Útangents)rB   r!   Ú_is_bwd_seed_offsetÚ_is_fwd_seed_offsetr   r   r   r   Ú
_is_primall   s    
ÿþürV   c                 C   s   | j dkod| jkS rR   ©rB   r!   r   r   r   r   Ú_is_tangentt   s    rX   c                 C   s   | j dkod| jkpd| jkS )Nr2   Zbwd_seedZbwd_base_offsetrW   r   r   r   r   rT   w   s    rT   c                 C   s   | j dkod| jkpd| jkS )Nr2   Zfwd_seedZfwd_base_offsetrW   r   r   r   r   rU   z   s    rU   )Újoint_modulec                C   s<   t  dd„ | jjD ƒ¡d }|d |… }||d … }||fS )Nc                 S   s   g | ]}|j d kr|j‘qS ©r?   )rB   rF   ©r6   r   r   r   r   r:      s     
 z,_extract_fwd_bwd_outputs.<locals>.<listcomp>r   )rD   rE   r   r   )rY   Únum_fwd_outputsrN   Úfwd_outputsÚbwd_outputsr   r   r   Ú_extract_fwd_bwd_outputs~   s    r_   c                C   s   t | |d�\}}ttt| jjƒƒ}ttt| jjƒƒ}ttt| jjƒƒ}ttt| jjƒƒ}	t	| j|| || | ƒ}
t	| j|| | |	 |ƒ}|jD ]\}|j
dkr’|js’|D ]}|j|jkrª| |¡  qÊqª|D ]}|j|jkrÎ| |¡  q’qÎq’tƒ }g }g }|D ]4}t|ƒ}|�r*| |¡ | |¡ n
| |¡ �qt| jƒ}t |||¡D ]d}d|jk�rd�qPt|jd ƒ| }t|dd„ d�D ]"}||k�r˜�q†| || ¡ �q†||O }�qP| ¡  | || ¡ t	| j|| || | ƒ}
t	| j|| | |	 |ƒ}t | |
¡}t | |¡}||fS )N©r\   r2   r&   c                 S   s   | j S r;   ©rA   )Úsr   r   r   r=   À   r>   z*_extract_fwd_bwd_modules.<locals>.<lambda>©Úkey)r_   ÚlistÚfilterrV   r   r   rX   rU   rT   rQ   rB   ÚusersrA   ÚremoveÚsetr   ÚaddrK   r	   Ú	itertoolsÚchainr   r   ÚsortedÚclearÚextendr4   ÚGraphModule)rY   Úsaved_valuesÚsaved_sym_nodesr\   r]   r^   Úprimal_inputsÚtangent_inputsÚfwd_seed_offset_inputsZbwd_seed_offset_inputsZ	fwd_graphZ	bwd_graphr   Zsaved_valueZ	saved_symZsaved_symbolsZsaved_sym_nodes_bindingZsaved_sym_nodes_derivedÚsymbolZsymbol_bindingsZnew_symbolsrb   Z
fwd_moduleZ
bwd_moduler   r   r   Ú_extract_fwd_bwd_modules…   sv    
ýý






ýýrw   )rY   Úreturnc                   s~  t | ƒrt| ||d�S ttt| jjƒƒ}ttt| jjƒƒ}|| }t| |d�\}}t	| j||ƒ}dd„ |jD ƒ‰ g }	g }
| jjD ]À}|j
ˆ kr�q€t|ƒr¤|
 |¡ q€d|jkrê|jdkrê|j}tdd„ |D ƒƒsÔt‚|D ]}|	 |¡ qØq€‡ fdd	„|jD ƒ}d|jk�r6td
d„ |D ƒƒ�r6|D ]}|
 |¡ �q"q€|	 |¡ q€tdd„ |	D ƒ ¡ ƒ}	tdd„ |
D ƒ ¡ ƒ}
t| |	|
|d�S )aÔ  
    Partitions the :attr:`joint_module` in a manner that closely resembles the
    behavior observed in the original ``.forward()`` and ``.backward()`` of the
    callable, i.e., the resulting forward graph contains those operators that
    are executed in the original ``.forward()`` callable passed to
    :func:`aot_function`.

    The default partitioner collects the operators that are between the forward
    inputs and the forward outputs. This helps in finding the tensors which have
    to be stashed for the backward pass. These stashed tensors become the output
    of the generated forward graph. The remaining operators are then placed in
    the backward graph.

    .. warning::
        This API is experimental and likely to change.

    Args:
        joint_module(fx.GraphModule): The joint forward and backward graph. This
            is the result of AOT Autograd tracing.

    Returns:
        Returns the generated forward and backward Fx graph modules.
    r`   c                 S   s   h | ]}|j d kr|j’qS rZ   ©rB   rA   r[   r   r   r   Ú	<setcomp>  s     
 z$default_partition.<locals>.<setcomp>Útensor_metar3   c                 s   s   | ]}|j tjkV  qd S r;   )r!   ÚoperatorÚgetitem©r6   Úuserr   r   r   Ú	<genexpr>  s     z$default_partition.<locals>.<genexpr>c                    s   g | ]}|j ˆ kr|‘qS r   ra   ©r6   Ún©Zforward_node_namesr   r   r:     s     
 z%default_partition.<locals>.<listcomp>c                 s   s   | ]}t |ƒV  qd S r;   ©r   r�   r   r   r   r€     s     c                 S   s   i | ]
}|d “qS r;   r   ©r6   Úkr   r   r   Ú
<dictcomp>&  s      z%default_partition.<locals>.<dictcomp>c                 S   s   i | ]
}|d “qS r;   r   r…   r   r   r   r‡   '  s      ©rr   r\   )r   Ú#min_cut_rematerialization_partitionre   rf   rV   r   r   rU   r_   rQ   rA   r   rK   r   rB   rg   Úallr*   Úkeysrw   )rY   Ú_joint_inputsr\   rs   ru   rM   r]   r^   Úforward_only_graphrq   rr   r   rg   r   Zbackward_usagesr   rƒ   r   Údefault_partitionâ   s>    
ÿþ 	rŽ   c                 C   s   d}| D ]}||9 }q|S ©Nr   r   )r7   rb   Úir   r   r   Ú_prod,  s    
r‘   c                 C   s
   | |j  S r;   )Úitemsize)ÚnumelÚdtyper   r   r   Ú_tensor_nbytes2  s    r•   )r   rx   c                 C   sÀ   d| j kr†| j d }t|tƒr4t|tjƒr.dS dS n@t|ttfƒrTtdd„ |D ƒƒS t|tjƒrtt	t
| ¡ ƒ|jƒS tdt|ƒ› �ƒ‚d| j kr²| j d }ttt|jƒƒ}|j}ndS t	||ƒS )	Nr&   r   i?B c                 s   s.   | ]&}t |tjƒrtt| ¡ ƒ|jƒV  qd S r;   )r(   r"   ÚTensorr•   r   r“   r”   r�   r   r   r   r€   >  s      z_size_of.<locals>.<genexpr>zUnknown metadata type r{   r   )r   r(   r   r"   r)   re   ÚtupleÚsumr–   r•   r   r“   r”   rJ   Útyper‘   ÚmapZto_size_hintÚshape)r   r&   Úmetadatar“   r”   r   r   r   Ú_size_of5  s"    




r�   c                 C   s\   ddl m} |tƒ}| jD ]"}|jdkr||jj  d7  < qtt| 	¡ dd„ dd�ƒ d S )	Nr   r
   r3   r   c                 S   s   | d S r�   r   r<   r   r   r   r=   V  r>   z_count_ops.<locals>.<lambda>T©rd   Úreverse)
Úcollectionsr   Úintr   rB   r!   r/   Úprintrm   Úitems)r   r   Zcntr   r   r   r   Ú
_count_opsP  s    

r¤   c                  C   sl   g } t tjjƒD ]V}ttjj|ƒ}t|tjjƒs2q| ¡ D ]*}t||ƒ}tj	j
|jkr:|  |¡  qq:q| S r;   )Údirr"   ÚopsÚatenÚgetattrr(   Z_opsZOpOverloadPacketZ	overloadsr#   Z	pointwiser   rK   )r¦   Ú	attr_nameZopoverloadpacketÚoverloadZop_overloadr   r   r   Úpointwise_opsY  s    

r«   c                    s¤   | ˆ krˆ |  S | j dkr*dˆ | < ˆ |  S | j dkrf| jd }|D ]}t|tjjjƒrBt|ˆ ƒ qBd S ‡ fdd„| jD ƒ}t	|ƒdkrŒdg}t
|ƒd ˆ | < ˆ |  S )Nr2   r   r?   c                    s&   g | ]}t |tjjjƒrt|ˆ ƒ‘qS r   )r(   r"   r4   r   r5   Ú	get_depth©r6   Úarg©Ú	depth_mapr   r   r:   |  s      zget_depth.<locals>.<listcomp>r   )rB   rF   r(   r"   r4   r   r5   r¬   Úall_input_nodesÚlenÚmax)r   r°   rF   r®   Ú
arg_depthsr   r¯   r   r¬   j  s     


r¬   c                    s(   ‡ fdd„| D ƒ}t | ¡ dd„ dd�S )Nc                    s&   i | ]}t |tjjjƒr|ˆ | “qS r   )r(   r"   r4   r   r5   r­   r¯   r   r   r‡   …  s       zsort_depths.<locals>.<dictcomp>c                 S   s   | d S r�   r   r<   r   r   r   r=   †  r>   zsort_depths.<locals>.<lambda>Trž   )rm   r£   )rF   r°   r´   r   r¯   r   Úsort_depths„  s    rµ   c                    s(  t  ¡ ‰i ‰| jjD ]*}|jdkrˆ |j¡}|j|_|ˆ|< qi }t| jjƒD ]\}}|||< qPi ‰ dd„ | jjD ƒd }t	|ˆ ƒ ‡ ‡‡‡fdd„‰t
tt| jjƒƒ}d}tj}|D ](}	|	jD ]}
||
 |k rÂ||
 }|
}qÂq¸|dk	sît‚t
| jjƒ|| d… D ]}ˆ|ƒ �qtj  | ˆ¡}|S )aÃ  
    This pass finds the first bwd node in the graph (by looking at users of
    tangents) and then reorders the graph by walking from this node to all the
    way to the end of the graph. At each op in this traveral, we insert this op
    in a new graph and try to bring only the relevant subgraph from the other
    non-bwd edges relevant for this op. This closely mimics the behavior of
    autograd engine.

    Why is this pass required in the first place?

    This is an artifact of how partitioners work today. The starting point of
    partitioner is a joint graph, which is fwd and then bwd graph. In the case
    of checkpointing, we keep portions of fwd graph in their original place in
    the joint graph, while obtaining a bwd graph. As a result, the resulting bwd
    graph has copies of recomputed fwd subgraphs followed by the original bwd
    graph. If we run this naively, this leads to bad memory footprint, because
    the fwd subgraphs are live for way longer duration than necessary. This pass
    reorders the operations such that we prioritize the ops for the original bwd
    graph while only realizing those ops from the fwd graph that are necessary
    at any given point in the graph.
    r2   c                 S   s   g | ]}|j d kr|‘qS rZ   ©rB   r[   r   r   r   r:   ²  s     
 z7reordering_to_mimic_autograd_engine.<locals>.<listcomp>r   c                    sR   | ˆkrˆ|  S t | jˆ ƒD ]\}}ˆ|ƒˆ|< qˆ | ‡fdd„¡ˆ| < ˆ|  S )Nc                    s   ˆ |  S r;   r   r<   r8   r   r   r=   ½  r>   zSreordering_to_mimic_autograd_engine.<locals>.insert_node_in_graph.<locals>.<lambda>)rµ   r±   rI   )r   r®   Ú_©Zdepthsr9   Úinsert_node_in_graphrO   r   r   r¹   µ  s    zAreordering_to_mimic_autograd_engine.<locals>.insert_node_in_graphN)r4   r@   r   r   rB   r2   rA   r   Ú	enumerater¬   re   rf   rX   ÚmathÚinfrg   r*   r"   rp   )Zgmr   rP   ÚorderÚidxZoutput_nodert   Zfirst_node_in_bwdZminimum_orderÚtangentr   Znew_gmr   r¸   r   Ú#reordering_to_mimic_autograd_engine‰  s6    




rÀ   c               
   C   s’  t  ¡ }dd„ }dd„ }dd„ }|| ƒ}||ƒ}	||ƒ}
tƒ }| jjD ]T}t|ƒrFt|jdƒrFtj	j
|jjkrF||j }|	|j }|
|j }||dœ||< qFtjjj}tjjj}|jjD ] }|jd	kr¸d
|jkr¸|} qÚq¸g }| ¡ D �]0\}}|d }|d }|j}| |¡�r |jd||jf|j˜|jd�}|jdtj|dfi d�}|jdtj|dfi d�}| |¡ | |¡ | |¡ W 5 Q R X |j}| |¡�0 dt|ƒ› �}| |¡}|||ƒƒ|jd< W 5 Q R X | |¡�: |jd|||jf|j˜|jd�}| |¡ | |¡ W 5 Q R X qædd„ |jjD ƒd }|jd }t|ƒ| }|d |… | ||d …  }|j  |¡ |j |¡ | !¡  | !¡  ||fS )Nc                 S   sF   i }| j jD ]4}|jdkrt|jdƒrtjj|jjkr|||j	< q|S )Nr3   r   )
r   r   rB   r    r!   r"   r#   r$   r   rA   )ZgmodZrandom_nodesr   r   r   r   Úget_rng_opsê  s    ÿ
þýz*functionalize_rng_ops.<locals>.get_rng_opsc                 S   sT   d| j krdS | j d }t|tƒs(|f}|D ]"}t|tjƒr,|jjdkr, dS q,dS )zV
        Check the example value of the node outputs to find the device type.
        r&   NÚcudaÚcpu)r   r(   r—   r"   r–   Údevicer™   )r   Ú
candidatesÚ	candidater   r   r   Ú
get_deviceõ  s    


z)functionalize_rng_ops.<locals>.get_devicec                 S   s   | dkrt j ¡ S t  ¡ S )NrÂ   )r"   rÂ   Zget_rng_state)rÄ   r   r   r   Úget_sample_rng_state  s    
z3functionalize_rng_ops.<locals>.get_sample_rng_stater   )ÚfwdÚbwdr2   r¿   rÉ   rÊ   r3   )rF   rG   r   r   Zrng_state_output_r&   c                 S   s   g | ]}|j d kr|‘qS rZ   r¶   r[   r   r   r   r:   O  s     
 z)functionalize_rng_ops.<locals>.<listcomp>)"rk   ÚcountÚdictr   r   r   r    r!   r"   r#   r$   r   rA   Z_primsZ	rng_primsZrun_and_save_rng_stateÚrun_with_rng_staterB   r£   Zinserting_beforeZcreate_noderF   rG   r|   r}   Zreplace_all_uses_withZ
erase_noderK   Únextr2   r   r²   r?   Ú	recompile) rY   Ú	fw_moduleÚ	bw_moduleZnum_sym_nodesÚuidrÁ   rÇ   rÈ   Zjoint_graph_rng_opsZfw_graph_rng_opsZbw_graph_rng_opsZrecomputable_rng_ops_mapr   Z	base_nodeZfw_nodeZbw_nodeZrun_and_save_rngrÍ   Zbw_tangent_start_nodeZfw_rng_state_outputsZ	node_pairZfw_graphZfunctional_fw_nodeÚstateZ
rng_outputZbw_graphZ
state_nameZbw_rng_state_nodeZfw_output_nodeZ
fw_outputsZsym_node_start_idxrN   r   r   r   Úfunctionalize_rng_opsÔ  s‚    ÿ
þý




ü


ü

rÔ   c                 C   sL   | j jD ]>}t|ƒr|jD ]*}t|ƒr|jd |jd krd|jd< qq| S )a  
    If there are two consecutive checkpointed blocks with no operator in
    between, we would still want to stash the tensor at the boundary of
    checkpointed blocks. The following pass makes the last output node
    non-recomputable to allow for that.
    r   r   )r   r   r   rg   r   )rY   r   r   r   r   r   Úcleanup_recompute_tagsZ  s    
rÕ   Úinductorc          ,   K      s  zddl }W n, tk
r8 } ztdƒ|‚W 5 d}~X Y nX | j ¡  |  ¡  | j}tjrft|ƒ}|| _| j}	t	| ƒ‰t
| ƒ}
ˆrˆt| ƒ} i ‰| jjD ]}|ˆ|j< q”‡‡fdd„}|| ƒ\}‰}}t|ƒdkrÜt| |ˆd�S t| jjƒD ]@}|ˆkrüd|_n,tdƒ|_|jD ]}t|j|jd ƒ|_�qqètjj‰tjj}ˆjˆjˆjˆjˆjˆjˆjˆjˆj ˆj!ˆj"ˆj#ˆj$ˆj%ˆj&ˆj'ˆj(ˆj)ˆj*ˆj+ˆj,ˆj-ˆj.ˆj/ˆj0ˆj1ˆj2ˆj3ˆj4ˆj5ˆj6ˆj7ˆj8ˆj9ˆj:ˆj;ˆj<ˆj=ˆj>ˆj?ˆj@ˆjAˆjBˆjCˆjDˆjEˆjFˆjGˆjHˆjIˆjJˆjKˆjLˆjMˆjNˆjOˆjPˆjQˆjRˆjSˆjTˆjUˆjVˆjWˆjXˆjYˆjZˆj[ˆj\ˆj]t^j_ˆj`ˆjaˆjbˆjcgK}ˆj`ˆjaˆjdg‰ˆd	k�r<||j|jeˆjfˆjcˆjg|jh|jWˆjhˆji|jjˆjkˆjlˆjmˆjnˆjoˆjpˆjqˆjrˆjsˆjtˆjuˆjvˆjwˆjxˆjyˆjzˆj{ˆj|ˆj}ˆj~ˆjˆj€g 7 }ˆˆjnˆjpˆjlˆj�|jjˆjoˆj}g7 ‰|ˆj‚g7 }|ˆ7 }|tƒƒ 7 }|ˆj„g7 }|d
d„ t…D ƒ7 }ˆdk	�r~t†ˆƒnt†|ƒ‰ˆj‡ˆjˆˆj‰g}ˆjŠˆj‹ˆjŒˆj�ˆjŽˆj�ˆj�ˆj‘ˆj’ˆj“ˆj”ˆj•ˆj–g}|| ‰ˆt†|ƒB ‰t—�rdd„ | jjD ƒ}|dd„ ˆD ƒ }t˜d|ƒ t˜ƒ  d‰ ‡‡‡fdd„‰‡ ‡‡‡‡‡‡fdd„}‡fdd„‰‡fdd„‰tdœ‡fdd„}| ™¡ ‰|	jD �]>}|jšdk�r–�q€||k�r¼ˆj›|jd dtœj�d� �q€tž|ƒ�sÐtŸ|ƒ�rèˆj›d |jd tœj�d� ||ƒ�r|ˆk�rˆj›d |jd tœj�d� d!|j k�o(d"|j k�pJd!|j k�oJt¡|j d! tj¢ƒ }t£|ƒ�r`t¤|ƒ}n|�rntœj�}n||ƒ}ˆj›|jd |jd# |d� |jD ]$}ˆj›|jd# |jd tœj�d� �q˜�q€z| ¥ˆd d¡\}}W n8 t¦k
�r   t˜d$ƒ t˜d% §|j¨j© ªˆ¡¡ƒ ‚ Y nX |\}‰
t†ƒ }‡fd&d'„|D ƒD ]$\‰}| «‡
‡fd(d'„|D ƒ¡ �q2t†ƒ } |D ]>\}!}"|!dd)… |"dd*… k�sˆt¬‚|!dd)… }#|  |#¡ �qbd+d,„ t­| jjƒD ƒ‰	t®‡fd-d'„| D ƒ‡	fd.d/„d0�}$t¯t°d1d/„ |$ƒƒ}%t¯t°d2d/„ |$ƒƒ}$t±| |$|%ˆd3�\}&}'ˆ�r<|
�r4t²| |&|'t|%ƒƒ\}&}'t³|'ƒ}'t—�rt˜d4tWd5d„ |$D ƒƒd ƒ d6d„ |&jjD ƒ}(d7d„ |'jjD ƒ})|(|)@ }*t´tƒ}+|&jjD ]8}|j|*k�rÐtµ|j¶d8ƒ�rÐ|+t·|j¶j¸ƒ  d7  < �qšt˜d9t|*ƒ› d:t|(ƒ› d:t|)ƒ› �ƒ t˜d;t®|+ ¹¡ d<d/„ d=d>�ƒ |&|'fS )?ay  
    Partitions the joint graph such that the backward recomputes the forward.
    Recomputing helps in trading off memory bandwidth with computation.

    To create the fwd and bwd graph, we copy the joint graph, manually set the
    outputs to just original forward or backward outputs. And then we run the
    resulting graphs through dead code elimintation.

    .. warning::
        This API is experimental and likely to change.

    Args:
        joint_module(fx.GraphModule): The joint forward and backward graph. This
            is the result of AOT Autograd tracing.
        _joint_inputs: The inputs to the joint graph. This is unused.
        compiler: This option determines the default set of recomputable ops.
            Currently, there are two options: ``nvfuser`` and ``inductor``.
        recomputable_ops: This is an optional set of recomputable ops. If this
            is not None, then this set of ops will be used instead of the
            default set of ops.
        num_fwd_outputs: The number of outputs from the forward graph.

    Returns:
        Returns the generated forward and backward Fx graph modules.
    r   NzANeed networkx installed to perform smart recomputation heuristicsc           
         sæ   t ƒ ‰ | jjD ]@}|jdkr0d|jkr0ˆ  |¡ |ˆ kr|jD ]}ˆ  |¡ q>qttt	| jjƒƒ}ttt
| jjƒƒ}|| }t| ˆd�\}}ˆ  dd„ |D ƒ¡ t| j||ƒ}‡fdd„|jD ƒ‰‡ ‡fdd„| jjD ƒ}	|ˆˆ |	fS )	Nr2   rS   r`   c                 s   s   | ]}|d k	r|V  qd S r;   r   )r6   Úor   r   r   r€   ­  s      zNmin_cut_rematerialization_partition.<locals>.classify_nodes.<locals>.<genexpr>c                    s    h | ]}|j d krˆ |j ’qS rZ   ry   r[   ©Úname_to_noder   r   rz   ¯  s    
ÿzNmin_cut_rematerialization_partition.<locals>.classify_nodes.<locals>.<setcomp>c                    s    h | ]}|ˆkr|ˆ kr|’qS r   r   r[   ©Úrequired_bw_nodesÚrequired_fw_nodesr   r   rz   ±  s     ÿ)ri   r   r   rB   r!   rj   rg   re   rf   rV   rU   r_   ÚupdaterQ   )
rY   r   r   rs   ru   rM   r]   r^   r�   Úunclaimed_nodes)rÙ   r\   rÚ   r   Úclassify_nodes   s     

z;min_cut_rematerialization_partition.<locals>.classify_nodesr`   g    eÍÍAr   rÖ   c                 S   s   g | ]}t |ƒ‘qS r   )r   )r6   Úmr   r   r   r:   Ø  s   ÿz7min_cut_rematerialization_partition.<locals>.<listcomp>c                 S   s.   h | ]&}|j d krt|jdƒrt|jjƒ’qS )r3   Ú_overloadpacket)rB   r    r!   Ústrrá   r[   r   r   r   rz   æ  s   
 þz6min_cut_rematerialization_partition.<locals>.<setcomp>c                 S   s   h | ]}t |ƒ’qS r   )râ   ©r6   r�   r   r   r   rz   ë  s     z#Ops banned from rematerialization: Fc                    sb   | h}t |ƒdkr^| ¡ }|jD ]:}|ˆkr<ˆ ||ƒs< dS |ˆkr t|ƒˆkr | |¡ q qdS )Nr   TF)r²   Úpoprg   r   rj   )r   Z	cur_nodesÚcurr   )Ú
is_fusiblerÜ   Úview_opsr   r   Úis_materialized_backwardsñ  s    
zFmin_cut_rematerialization_partition.<locals>.is_materialized_backwardsc                    sÎ   d| j kr| j d dkS ˆ r2| jdko0t| ƒˆkS | jdkr@dS t| ƒˆkrPdS | jtjkr`dS | jˆjjˆjjfkrzdS ˆ| ƒr†dS ˆs¢ˆdkr¢| j	t
jkr¢dS tdd„ | jD ƒƒ}t| ƒ}|d	 |k S d S )
Nr   r   r3   FTrÖ   c                 s   s"   | ]}t |tjƒrt|ƒV  qd S r;   )r(   r4   r5   r�   rã   r   r   r   r€      s      zQmin_cut_rematerialization_partition.<locals>.ban_recomputation.<locals>.<genexpr>r'   )r   rB   r   r!   r|   r}   Úlift_fresh_copyÚdefaultZ
lift_freshÚdist_from_bwr   Zmax_dist_from_bwr˜   rF   r�   )r   Zinput_tensors_sizeZoutput_size)ÚAGGRESSIVE_RECOMPUTATIONr§   ÚcompilerÚgraph_has_recomputable_opsrè   Úrecomputable_opsÚunrecomputable_opsr   r   Úban_recomputationý  s(    

	z>min_cut_rematerialization_partition.<locals>.ban_recomputationc                    s   t | ƒˆ kot |ƒˆ kS r;   )r   )ÚaÚb)Úfusible_opsr   r   ræ   $  s    z7min_cut_rematerialization_partition.<locals>.is_fusiblec                    s*   ˆ j dkrdS t‡‡ fdd„ˆ jD ƒƒ S )Nr2   Tc                 3   s   | ]}ˆ ˆ|ƒV  qd S r;   r   r~   )ræ   r   r   r   r€   +  s     zOmin_cut_rematerialization_partition.<locals>.is_materialized.<locals>.<genexpr>)rB   rŠ   rg   r   )ræ   r   r   Úis_materialized'  s    
z<min_cut_rematerialization_partition.<locals>.is_materialized)rx   c                    s>   t | ƒ}t|dtt| jdƒdƒ  ƒ}ˆ | ƒr2|S |d S d S )Ngš™™™™™ñ?éd   r   é   )r�   r¡   r³   Úminrë   )r   Zmem_sz)rõ   r   r   Úget_node_weight-  s
    z<min_cut_rematerialization_partition.<locals>.get_node_weightr?   Z_inZsink)ÚcapacityÚsourcer&   r{   Z_outz-Failed to compute min-cut on following graph:Ú
c                 3   s   | ]}|ˆ | fV  qd S r;   r   r�   )Únx_graphr   r   r€   e  s     z6min_cut_rematerialization_partition.<locals>.<genexpr>c                 3   s   | ]}|ˆ krˆ|fV  qd S r;   r   )r6   Úv)Únon_reachableÚur   r   r€   f  s      éýÿÿÿéüÿÿÿc                 S   s   i | ]\}}||“qS r   r   )r6   r¾   r   r   r   r   r‡   o  s      z7min_cut_rematerialization_partition.<locals>.<dictcomp>c                 3   s   | ]}ˆ | V  qd S r;   r   r[   rØ   r   r   r€   p  s     c                    s   ˆ |  S r;   r   r<   )Únode_idxr   r   r=   p  r>   z5min_cut_rematerialization_partition.<locals>.<lambda>rc   c                 S   s   t | ƒS r;   r„   ©r‚   r   r   r   r=   r  r>   c                 S   s
   t | ƒ S r;   r„   r  r   r   r   r=   s  r>   rˆ   z Theoretical Activations Stored: c                 S   s   g | ]}t |ƒ‘qS r   )r�   rã   r   r   r   r:   €  s     c                 S   s   h | ]}|j d kr|j’qS ©r3   ry   r[   r   r   r   rz   �  s     
 c                 S   s   h | ]}|j d kr|j’qS r  ry   r[   r   r   r   rz   ‚  s     
 rá   z# remat/fw/bw: ú/zCount of Ops Rematerialized: c                 S   s   | d S r�   r   r<   r   r   r   r=   Š  r>   Trž   )ºZnetworkxÚImportErrorrJ   r   rL   rÏ   r   Zcser   r   r%   rÕ   r   rA   r²   rŽ   Úreversedrë   r¡   rg   rø   r"   r¦   r§   Úprimsrj   ÚsubÚdivÚatan2Úmulr³   ÚpowÚ	remainderÚfmodÚ__and__Ú__or__Ú__xor__Ú
__lshift__Ú
__rshift__ÚeqÚneÚgeÚgtÚleÚltÚabsZbitwise_notÚceilÚfloorÚfracÚnegZreluÚroundZsiluÚtruncÚlogÚlog10Úlog1pÚlog2ÚlgammaÚexpÚexpm1ÚerfÚerfcÚcosÚacosÚcoshÚsinÚasinÚsinhÚtanÚatanÚtanhÚatanhÚsqrtZrsqrtZ
reciprocalZsigmoidZsoftplusÚ	thresholdZthreshold_backwardÚclampÚwhereZlerpZaddcmulZgeluZgelu_backwardr˜   ZmeanZ_grad_sum_to_sizeZsum_to_sizeZamaxÚtoZtype_asr|   r}   ZsqueezeZ	unsqueezeZrsubZ_to_copyÚaliasZconvert_element_typeÚcloneZ	full_likeÚvarZstdZbroadcast_in_dimÚselectZpermuteZ_unsafe_viewÚviewÚexpandÚsliceZreshapeZbroadcast_tensorsZscalar_tensorZonesZ	new_zerosré   ZarangeZtriuZvar_meanÚisinfrH   ÚfullZ
as_stridedZzerosZargmaxÚmaximumÚtÚindexr«   Z
zeros_liker   ri   Znative_dropoutZ	rand_likeZ
randn_likeÚmmZconvolutionZconvolution_backwardZbmmZaddmmZupsample_bilinear2dZ_softmaxZ_softmax_backward_dataZnative_layer_normZnative_layer_norm_backwardZnative_batch_normZnative_batch_norm_backwardZ_native_batch_norm_legitÚAOT_PARTITIONER_DEBUGr¢   ZDiGraphrB   Zadd_edger»   r¼   rV   rU   r   r(   r–   r   r+   Zminimum_cutÚ	ExceptionÚjoinZ	readwriteZedgelistZgenerate_edgelistrÝ   r*   rº   rm   re   rf   rw   rÔ   rÀ   r   r    r!   râ   rá   r£   ),rY   rŒ   rí   rï   r\   ÚnxÚer   Z	cse_graphZfull_bw_graphZgraph_has_recomputable_rng_opsr   rß   Zorig_fw_outputsrÛ   rÞ   r   r	  Zdefault_recomputable_opsZ
random_opsZcompute_intensive_opsZjoint_module_opsZops_ignoredrñ   rù   Zis_non_tensor_nodeÚweightZ	cut_valueÚ	partitionZ	reachableZcutsetZnbrsZ	cut_nodesZnode_inZnode_outZ	node_namerq   rr   rÐ   rÑ   Zfw_module_nodesZbw_module_nodesZremat_nodesÚcountsr   )rì   r§   rí   rô   rî   ræ   rõ   rè   rÙ   r  rÿ   r\   rý   rï   rÜ   r   rð   rç   r   r‰   i  s
   ÿ


ÿ 1
ˆ$
ÿþ8þ
'
ÿ


&"   ÿ
   ÿ&r‰   Úfx_graphT)ÚtracedÚfnameÚfignamec           
      C   s’   |r0t  | j¡}t | |¡} | jjD ]
}i |_q$tj 	|¡\}}|sHd}t
d|› |› �ƒ t | |¡}| ¡ }	t|	d| d¡ ƒ|› |› �ƒ d S )Nz.svgzWriting FX graph to file: Zwrite_Ú.)ÚcopyÚdeepcopyr   r4   rp   r   r   ÚosÚpathÚsplitextr¢   r   ZFxGraphDrawerZget_main_dot_graphr¨   Úlstrip)
rQ  rR  rS  Z
clear_metarO   r   ÚbaseÚextÚgr7   r   r   r   Ú
draw_graphŽ  s    r^  úfull_graph.pngc                 C   s   t | |ƒ t| |ƒS r;   )r^  rŽ   )r   Zjoint_inputsÚ	file_namer   r   r   Údraw_joint_graph�  s    
ra  )rÖ   N)rP  T)r_  )EZ"torch.fx.experimental.proxy_tensorr   r   Z%torch.fx.experimental.symbolic_shapesr   r   r   r   r   r	   r"   Ztorch.fxr4   r|   r»   Ztorch.utils._pytreeÚutilsZ_pytreerD   rU  rW  rk   Zsympyr    r   Ztorch.fx.passesr   Útypingr   Zcompile_utilsr   r   Ú r   Ú	functoolsZdebug_partitionerrH  r   r   r%   r+   r,   rC   rQ   rV   rX   rT   rU   rp   r_   rw   rŽ   r‘   r•   r5   r¡   r�   r¤   Ú	lru_cacher«   r¬   rµ   rÀ   rÔ   rÕ   r‰   râ   r^  ra  r   r   r   r   Ú<module>   sn    5^þJ	
K    ÿý  '