U
    ªÊ¹jà7  ã                   @   sÒ  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlmZmZ d dlmZ d dlmZmZmZ d dlZd dlmZ d dlmZmZ d dlmZ d dlmZ d d	lm Z m!Z! d d
l"m#Z# d dl$m%Z% ddl&m'Z'm(Z( ddl)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0 e 1e2¡Z3e 4d¡dd„ ƒZ5d$dd„Z6ee* ej7dœdd„Z8ej9dd„ ƒZ:G dd„ dƒZ;ee Z<G dd„ dƒZ=ej>G dd„ dƒƒZ?e @¡ ZAd d!„ ZBd"d#„ ZCdS )%é    N)ÚAnyÚList)Úpatch)Ú
draw_graphÚget_aot_graph_nameÚget_graph_being_compiled)Úfx)Úsave_graph_reproÚwrap_compiler_debug)Úget_debug_dir)ÚGraphModule)Ú_extract_tensor_metadataÚTensorMetadata)Úlegalize_graph)Útree_mapé   )ÚconfigÚir)ÚBaseSchedulerNodeÚFusedSchedulerNodeÚNopKernelSchedulerNodeÚ
OutputNodeÚSchedulerNode)ÚVc                   C   s8   zt jddgt jd� W dS  t jk
r2   Y dS X d S )NÚwhichÚdot)ÚstderrTF)Ú
subprocessÚcheck_outputÚPIPEÚSubprocessError© r!   r!   úT/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/_inductor/debug.pyÚhas_dot*   s
    r#   Fc           	   	   C   sÐ   t ƒ st d¡ dS |dkr"tƒ }t| ƒ}|jD ]d}d|jkr@q0|jd j}t|t	ƒr^|d }d}t|t
jƒrv|jj}t||dddddƒ}||jd< q0|r¢t|ƒ ti |ƒ}t|ƒ |j ¡  t||dd� dS )zR
    Draw a graph in fname.svg.
    nodes is a list of SchedulerNode objects.
    z*draw_buffers() requires `graphviz` packageNÚfusion_metar   Ztensor_metaF)Z
clear_meta)r#   ÚlogÚwarningr   Úcreate_fx_from_snodesÚnodesÚmetaÚgroupÚ
isinstanceÚtupler   ZComputedBufferÚdataÚdtyper   Úprintr   r   ÚgraphZlintr   )	r(   Zprint_graphÚfnamer0   Únoder*   r.   ÚmetadataÚgmr!   r!   r"   Údraw_buffers3   s.    





r5   )ÚsnodesÚreturnc                    s  dd„ }t  ddddg¡}i }tj ¡ }d}g }d}| D �]$}| ¡ rPd}	|	}nZ| ¡ rbd	}	|	}nHt|tƒrvd
}	|	}n4t|t	ƒrŒd}	|j
}nt|tƒr¢d}	|j
}ntdƒ‚tjj | ¡ d¡}
|	› d|
› �}||ƒ}|j|ddd�}‡ fdd„‰ ˆ |ƒ�r| |¡ | ¡ }||_||||	ƒ|jd< t|tƒ�rJ|jD ]}||| ¡ < �q6|||< |dkr8|}q8| D ]†}| ¡ }|jj}|| }g }|D ]T}|j|k�r¤||j }n,| |¡� | |j¡}|||j< W 5 Q R X | |¡ �qˆt|ƒ|_�qd| t|ƒdk�r|d nt|ƒ¡ |S )zB
    Creates a FX Graph from a list of SchedulerNode objects.
    c                 S   s   dd„ }| |_ |S )Nc                  W   s   dS )Nr   r!   )Úargsr!   r!   r"   Úfunc1_   s    z;create_fx_from_snodes.<locals>.get_fake_func.<locals>.func1)Ú__name__)Únamer9   r!   r!   r"   Úget_fake_func^   s    z,create_fx_from_snodes.<locals>.get_fake_funcÚ
FusionMetar*   ÚsnodeÚtypeNZexternÚtemplateZnopZcomputeZfusedzUnknown node typeZoriginal_atenz: r!   ©r8   Úkwargsc                    s6   t | tƒr"t‡ fdd„| jD ƒƒS tdd„ | jD ƒƒS )Nc                 3   s   | ]}ˆ |ƒV  qd S ©Nr!   )Ú.0Úx©Ú	in_outputr!   r"   Ú	<genexpr>Š   s     z;create_fx_from_snodes.<locals>.in_output.<locals>.<genexpr>c                 s   s   | ]}t |jtƒV  qd S rC   )r+   r2   r   )rD   Úuserr!   r!   r"   rH   ‹   s     )r+   r   Úanyr6   Zusers)r>   rF   r!   r"   rG   ˆ   s    
z(create_fx_from_snodes.<locals>.in_outputr$   r   r   )ÚcollectionsÚ
namedtupleÚtorchr   ÚGraphZ	is_externZis_templater+   r   r   r*   r   ÚRuntimeErrorZ	_inductorÚutilsZget_fused_kernel_nameZ	get_nodesZcall_functionÚappendÚget_namer;   r)   r6   Zread_writesZreadsZinserting_beforeÚplaceholderr,   r8   ÚoutputÚlen)r6   r<   r=   Zbuf_to_fx_noder0   Z
first_nodeÚoutputsr*   r>   Z	node_typeZ
fused_nameÚ	func_nameZ	node_funcZfx_noder;   rE   ÚdepsÚnew_argsÚdepZdep_noder!   rF   r"   r'   Y   st    




 ÿ


$r'   c               	   c   sî   t j dd¡dk} dd l}t |jjj¡}t	 
¡ }| sPz
d V  W 5 | ¡  X d S | tddƒ¡ t j tƒ d¡}t j |¡s†t  |¡ t t j |dtƒ › d	�¡¡}| tj¡ | t d
¡¡ | |¡ z
d V  W 5 | |¡ | ¡  X d S )NZTORCH_COMPILE_DEBUGÚ0Ú1r   z*functorch.compile.config.debug_partitionerTÚtorchinductorZaot_z
_debug.logú3[%(filename)s:%(lineno)d %(levelname)s] %(message)s)ÚosÚenvironÚgetZtorch._functorch.aot_autogradÚloggingÚ	getLoggerZ
_functorchZaot_autogradr:   Ú
contextlibÚ	ExitStackÚcloseÚenter_contextr   ÚpathÚjoinr   ÚexistsÚmakedirsÚFileHandlerr   ÚsetLevelÚDEBUGÚsetFormatterÚ	FormatterÚ
addHandlerÚremoveHandler)Zcompile_debugrM   r%   Ústackrh   Úfhr!   r!   r"   Úenable_aot_logging²   s8    


þÿÿ


ru   c                   @   s‚   e Zd Ze ¡ Zedd„ ƒZedd„ ƒZdd„ Z	e
dœdd	„Zd
d„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ ZdS )ÚDebugContextc                    s"   t  ˆ ¡‡ fdd„ƒ}t|dd�S )Nc               
      s(   t ƒ � ˆ | |ŽW  5 Q R £ S Q R X d S rC   )rv   rA   ©Úfnr!   r"   Úinnerá   s    z DebugContext.wrap.<locals>.innerÚinductor)Zcompiler_name)Ú	functoolsÚwrapsr
   )rx   ry   r!   rw   r"   Úwrapß   s    zDebugContext.wrapc                 C   sJ   t jD ]>}tj tƒ d| › d|› �¡}tj |¡st |¡ |  S qd S )Nr]   Ú.)rv   Ú_counterr_   rh   ri   r   rj   rk   )Zfolder_nameÚnÚdirnamer!   r!   r"   Úcreate_debug_dirè   s    
ý
zDebugContext.create_debug_dirc                 C   s   d | _ d | _t ¡ | _d S rC   )Ú_profÚ_pathrd   re   Ú_stack)Úselfr!   r!   r"   Ú__init__ô   s    zDebugContext.__init__)Únew_pathc                 C   st   | j s
d S | d¡st|ƒ‚tj |¡r2t |¡ zt | j |¡ || _ W n$ t	k
rn   t
 d| j |¡ Y nX d S )Nz.debugz(Failed to copy debug files from %s to %s)r„   ÚendswithÚAssertionErrorr_   rh   rj   ÚshutilÚrmtreeÚcopytreeÚOSErrorr%   r&   )r†   rˆ   r!   r!   r"   Úcopyù   s    

  ÿzDebugContext.copyc                 C   s    | j s
t‚ttj | j |¡dƒS )NÚw)r„   rŠ   Úopenr_   rh   ri   ©r†   Úfilenamer!   r!   r"   Úfopen  s    
zDebugContext.fopenc                 C   s   t j | j|¡S rC   )r_   rh   ri   r„   )r†   Úsuffixr!   r!   r"   r“     s    zDebugContext.filenamec              	   C   s€   t jjd k	r|dd l}| jst‚tj | jtj 	| j¡› d�¡}| 
|d¡� }|j| jtj 	| j¡d� W 5 Q R X t j |¡ d S )Nr   z.tar.gzzw:gz)Úarcname)r   ÚtraceÚ
upload_tarÚtarfiler„   rŠ   r_   rh   ri   Úbasenamer‘   Úadd)r†   r™   Ztar_fileÚtarr!   r!   r"   r˜     s    
 ÿ$zDebugContext.upload_tarc                    s´   t jr<t d¡‰ ˆ j}ˆ  tj¡ ‡ fdd„}| j ||¡ | j 	t
 | ¡¡ t jjsZd S |  tƒ ¡| _t jjr~|  dtj¡ t jjr”|  dtj¡ t jjr°t ¡ | _| j ¡  d S )Nztorch._dynamoc                    s   ˆ   | ¡ d S rC   )rm   )Úlevel©r%   r!   r"   Úreset_log_level!  s    z/DebugContext.__enter__.<locals>.reset_log_levelz	debug.logzinfo.log)r   Údebugrb   rc   r�   rm   rn   r…   Úcallbackrg   r   Zset_debug_handlerr—   Úenabledr‚   r   r„   Z	debug_logÚ_setup_log_captureZinfo_logÚINFOZcompile_profileÚcProfileZProfilerƒ   Úenable)r†   Z
prev_levelrŸ   r!   rž   r"   Ú	__enter__  s"    

zDebugContext.__enter__c                 C   sp   t  d¡}| j |  |¡¡}t  |¡}| |¡ | t  d¡¡ | 	|¡ | t
|j|ƒ¡ | j |j|¡ d S )Nztorch._inductorr^   )rb   rc   r…   rg   r”   ÚStreamHandlerrm   ro   rp   rq   Úminr�   r¡   rr   )r†   r“   r�   r%   ÚfdÚchr!   r!   r"   r£   5  s    


ÿ
zDebugContext._setup_log_capturec                 C   sF   | j r| j  ¡  |  ¡  | jr8|  ¡  t dtƒ | j¡ | j 	¡  d S )Nz%s debug trace: %s)
rƒ   ÚdisableÚ_save_profile_datar„   r˜   r%   r&   r   r…   rf   )r†   Úexc_typeÚexc_valÚexc_tbr!   r!   r"   Ú__exit__A  s    
zDebugContext.__exit__c              	   C   sl   | j  |  d¡¡ |  d¡�F}tj| j |d�}| ¡  | d¡ | d¡ | d¡ | d¡ W 5 Q R X d S )Nzcompile.profzcompile.stats)ÚstreamZcumtimeéd   Ztottime)	rƒ   Z
dump_statsr“   r”   ÚpstatsZStatsZ
strip_dirsZ
sort_statsZprint_stats)r†   rª   Ústatsr!   r!   r"   r­   K  s    


zDebugContext._save_profile_datac                 C   sZ   t jjrJtt j|ƒrJztt| ƒ|ƒW S  tk
rF   tjddd� Y qVX ndd„ }|S d S )Nz Ignoring exception in debug codeT)Úexc_infoc                  _   s   d S rC   r!   rA   r!   r!   r"   Úignored]  s    z)DebugContext.__getattr__.<locals>.ignored)r   r—   r¢   ÚgetattrÚDebugFormatterÚ	Exceptionr%   r&   )r†   r;   r·   r!   r!   r"   Ú__getattr__U  s    zDebugContext.__getattr__N)r:   Ú
__module__Ú__qualname__Ú	itertoolsÚcountr   Ústaticmethodr}   r‚   r‡   Ústrr�   r”   r“   r˜   r§   r£   r±   r­   r»   r!   r!   r!   r"   rv   Ü   s   



rv   c                   @   sŠ   e Zd Zdd„ Zejjeej dœdd„Z	ejjeej dœdd„Z
edœd	d
„Zedœdd„Zeedœdd„Zedœdd„Zdd„ ZdS )r¹   c                 C   s   |j | _ |j| _|| _d S rC   )r”   r“   Úhandler)r†   rÂ   r!   r!   r"   r‡   g  s    zDebugFormatter.__init__)r4   Úinputsc              	   C   sP   |   d¡�}t|||dƒ W 5 Q R X |   d¡�}| |jdd�¡ W 5 Q R X d S )Nzfx_graph_runnable.pyrz   zfx_graph_readable.pyF©Zprint_output)r”   r	   ÚwriteÚprint_readable©r†   r4   rÃ   rª   r!   r!   r"   Úfx_graphl  s    zDebugFormatter.fx_graphc              	   C   s,   |   d¡�}| |jdd�¡ W 5 Q R X d S )Nzfx_graph_transformed.pyFrÄ   )r”   rÅ   rÆ   rÇ   r!   r!   r"   Úfx_graph_transformeds  s    z#DebugFormatter.fx_graph_transformed)r(   c                 C   s   |   d|¡ d S )Nzir_pre_fusion.txt©Ú	_write_ir©r†   r(   r!   r!   r"   Úir_pre_fusiony  s    zDebugFormatter.ir_pre_fusionc                 C   s   |   d|¡ d S )Nzir_post_fusion.txtrÊ   rÌ   r!   r!   r"   Úir_post_fusion|  s    zDebugFormatter.ir_post_fusion)r“   r(   c              	   C   s<   |   |¡�(}|D ]}| | ¡ ¡ | d¡ qW 5 Q R X d S )Nz


)r”   rÅ   Z	debug_str)r†   r“   r(   rª   r2   r!   r!   r"   rË     s    zDebugFormatter._write_irc                 C   s   t ||  d¡d� d S )Nzgraph_diagram.svg)r1   )r5   r“   rÌ   r!   r!   r"   Úgraph_diagram…  s    zDebugFormatter.graph_diagramc                 C   s   t  ||  d¡¡ d S )Nzoutput_code.py)r‹   r�   r“   r’   r!   r!   r"   Úoutput_codeˆ  s    zDebugFormatter.output_codeN)r:   r¼   r½   r‡   rM   r   r   r   ÚTensorrÈ   rÉ   ÚSchedulerNodeListrÍ   rÎ   rÁ   rË   rÏ   rÐ   r!   r!   r!   r"   r¹   f  s    ÿr¹   c                   @   s    e Zd ZU eed< ejed< dS )ÚTensorMetadataHolderÚtensor_metadataÚdeviceN)r:   r¼   r½   r   Ú__annotations__rM   rÕ   r!   r!   r!   r"   rÓ   Œ  s   
rÓ   c            
   	   O   s¢   d}t j |¡st  |¡ dd„ }t|| |fƒ\}}d}|› d|› dttƒ› d�}t|dƒ�}t 	||f|¡ W 5 Q R X t
 tj¡ržd	|› d
|›d�}	t|	ƒ dS )zè
    This function is used to save arguments for a compile_fx_inner function call
    to the file system.  Later on one can replay the compile_fx_inner call
    with the saved arguments using load_args_and_run_compile_fx_inner.
    z/tmp/inductor_saved_argsc                 S   s$   t | tjƒrtt| ƒ| jƒS | S dS )zæ
        Pickle FakeTensor will result in error:
        AttributeError: Can't pickle local object 'WeakValueDictionary.__init__.<locals>.remove'

        Convert all Tensor to metadata. This may also makes pickle faster.
        N)r+   rM   rÑ   rÓ   r   rÕ   ©rE   r!   r!   r"   Úhandle_tensor   s    z5save_args_for_compile_fx_inner.<locals>.handle_tensorÚcompile_fx_innerú/Ú_z.pklÚwbz3
Arguments for a compile_fx_inner call is saved to z’. To replay the call,
run the following:

from torch._inductor.debug import load_args_and_run_compile_fx_inner
load_args_and_run_compile_fx_inner(z
)
        N)r_   rh   rj   Úmkdirr   ÚnextÚsave_args_cntr‘   ÚpickleÚdumpr%   ÚisEnabledForrb   rn   r/   )
r8   rB   ÚfolderrØ   Zargs_to_saveZkwargs_to_saveÚfn_namerh   ÚfÚmessager!   r!   r"   Úsave_args_for_compile_fx_inner•  s     
ÿÿrç   c                 C   s¢   ddl m} t| dƒ�}t |¡\}}W 5 Q R X dd„ }tjjdd�}|�N t 	dd	¡�6 t
|||fƒ\}}|||ŽW  5 Q R £ W  5 Q R £ S Q R X W 5 Q R X d S )
Nr   )rÙ   Úrbc                 S   s4   t | tƒr,tjj | jj| jj| jj	| j
¡S | S d S rC   )r+   rÓ   rM   Z_dynamoZtestingZrand_stridedrÔ   ÚshapeZstrider.   rÕ   r×   r!   r!   r"   rØ   É  s    
üz9load_args_and_run_compile_fx_inner.<locals>.handle_tensorT)Zallow_non_fake_inputsZ	save_argsF)Ztorch._inductor.compile_fxrÙ   r‘   rà   ÚloadrM   Z_subclassesZFakeTensorModer   r   r   )rh   rÙ   rå   r8   rB   rØ   Z	fake_moder!   r!   r"   Ú"load_args_and_run_compile_fx_innerÃ  s    rë   )FN)DrK   rd   r¥   Údataclassesr{   r¾   rb   r_   Úos.pathrà   r´   r‹   r   Útypingr   r   Zunittest.mockr   Zfunctorch.compiler   r   r   rM   r   Ztorch._dynamo.repro.after_aotr	   r
   Ztorch._dynamo.utilsr   Ztorch.fx.graph_moduler   Ztorch.fx.passes.shape_propr   r   Ztorch.fx.passes.tools_commonr   Ztorch.utils._pytreer   Ú r   r   Z	schedulerr   r   r   r   r   Zvirtualizedr   rc   r:   r%   Ú	lru_cacher#   r5   rN   r'   Úcontextmanagerru   rv   rÒ   r¹   Ú	dataclassrÓ   r¿   rß   rç   rë   r!   r!   r!   r"   Ú<module>   sT   


&Y
) &.