U
    •Ê¹je9  ã                   @   s(  d dl Z d dlZd dlmZmZmZmZmZmZ d dl	Z	d dl
Z
d dlZd dlmZ d dlZd dlmZmZ d dlZd dlZe e¡Zdd„ Zdd„ Zeƒ Zd	d
„ Zeeeedƒee fZ!dZ"dd„ Z#G dd„ deƒZ$dddœdd„Z%dd„ Z&dd„ Z'dd„ Z(dd„ Z)dZ*dZ+dd„ Z,dd „ Z-d!d"„ Z.dS )#é    N)ÚAnyÚDictÚListÚ
NamedTupleÚOptionalÚTuple)ÚNamedTemporaryFile)Ú_frames_fmtÚ_block_extrac                    sD   d‰ ‡ fdd„}t  |¡ ‡ ‡fdd„‰tj ˆ¡ ‡fdd„}|S )NTc                      s   d‰ d S )NF© r   )Úenabledr   úV/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/utils/viz/_cycles.pyÚdisable   s    z observe_garbage.<locals>.disablec                    sV   ˆsd S | dkrt  t j¡ n4| dkrRt ¡ ‰dg‰‡‡ ‡‡‡fdd„}t |¡ d S )NÚstartÚstopFc                     s¨   ˆd sdˆd< n€t  ˆ¡ d‰ zjˆd dkr6t ¡  ˆtjƒ tj ¡  t d¡ tj 	¡ }t ¡  tj 	¡ }||krˆt
 d|| ¡ W 5 d‰ X ˆd k	r¤ˆ| |ŽS d S )Nr   TFZ
generationé   z.CUDA Memory changed during GC, %d bytes freed.)ÚsysÚ
setprofileÚgcZcollectÚgarbageÚclearÚ	set_debugÚtorchÚcudaZmemory_allocatedÚloggerÚwarning)ÚargsÚkwargsÚbeforeÚafter)r   ÚinfoÚobserverÚ
orig_traceÚself_returnr   r   Ú
do_collect"   s$    






z8observe_garbage.<locals>.gc_callback.<locals>.do_collect)r   r   ZDEBUG_SAVEALLr   Ú
getprofiler   )Zphaser    r$   )r   r!   )r    r"   r#   r   Úgc_callback   s    z$observe_garbage.<locals>.gc_callbackc                      s   t j ˆ ¡ d S ©N)r   Ú	callbacksÚremover   )r&   r   r   r)   D   s    zobserve_garbage.<locals>.remove)ÚatexitÚregisterr   r(   Úappend)r!   r   r)   r   )r   r&   r!   r   Úobserve_garbage   s    
)r-   c                  C   s   ddd„} t | ƒ jd ƒS )Nc                    s   ‡ fdd„S )Nc                      s   ˆ S r'   r   r   ©Úxr   r   Ú<lambda>Y   ó    z+_get_cell_type.<locals>.f.<locals>.<lambda>r   r.   r   r.   r   ÚfX   s    z_get_cell_type.<locals>.fr   )N)ÚtypeÚ__closure__)r2   r   r   r   Ú_get_cell_typeW   s    
r5   c                    s  i ‰‡fdd„‰‡‡fdd„‰ ‡ fdd„}‡ fdd„}‡‡fd	d
„}‡‡fdd„}‡‡fdd„}‡ fdd„}‡‡fdd„}‡ ‡‡fdd„}‡ fdd„}	t |t|t|t|t|tj|tj|t|tj	|t
j|tj|	i}
tˆƒjD ]}||
krÚ|
| ƒ  qÚˆ ddƒ tˆtƒ�rˆ dƒ ˆS )a%  
    Return known information about references held by the given object.

    Returns a mapping from referents to lists of descriptions.  Note that there
    may be more than one edge leading to any particular referent; hence the
    need for a list.  Descriptions are currently strings.

    c                    s   ˆ   t|ƒg ¡ | ¡ d S r'   )Ú
setdefaultÚidr,   )ÚnameÚobj)Ú
referencesr   r   Úadd_referencei   s    z+annotated_references.<locals>.add_referencec                     s(   | D ]}t ˆ|ƒrˆ |tˆ|ƒƒ qd S r'   )ÚhasattrÚgetattr)ÚattrsÚattr©r;   r9   r   r   Ú	add_attrsl   s    
z'annotated_references.<locals>.add_attrsc                      s&   zˆ dƒ W n t k
r    Y nX d S )NÚcell_contents)Ú
ValueErrorr   ©rA   r   r   Úadd_cell_referencesq   s    z1annotated_references.<locals>.add_cell_referencesc                
      s   ˆ ddddddddd	ƒ	 d S )
NÚ__defaults__r4   Ú__globals__Ú__code__Ú__name__Ú
__module__Z__doc____qualname__Ú__annotations__Ú__kwdefaults__r   r   rD   r   r   Úadd_function_references{   s    ÷z5annotated_references.<locals>.add_function_referencesc                     s(   t ˆƒD ]\} }ˆ d| › d�|ƒ qd S )Nú[ú])Ú	enumerate)ÚpositionÚitemr@   r   r   Úadd_sequence_referencesˆ   s    z5annotated_references.<locals>.add_sequence_referencesc                     s6   ˆ  ¡ D ](\} }ˆ d| ƒ ˆ dt| ƒ› d�|ƒ qd S )NÚkeyrN   rO   )ÚitemsÚrepr)rT   Úvaluer@   r   r   Úadd_dict_referencesŒ   s    
z1annotated_references.<locals>.add_dict_referencesc                     s   ˆD ]} ˆ d| ƒ qd S )NÚelementr   )Úeltr@   r   r   Úadd_set_references‘   s    z0annotated_references.<locals>.add_set_referencesc                      s   ˆ dddƒ d S )NÚ__self__Ú__func__Zim_classr   r   rD   r   r   Úadd_bound_method_references•   s    z9annotated_references.<locals>.add_bound_method_referencesc                     s:   t ˆƒtjkr6t ˆ¡} t| ƒdkr6| d }ˆ d|ƒ d S )Né   r   Ú__callback__)r3   ÚweakrefÚrefr   Úget_referentsÚlen)Z	referentsÚtargetr@   r   r   Úadd_weakref_references˜   s
    
z4annotated_references.<locals>.add_weakref_referencesc                     sL   ˆj } ˆ ddddddƒ t| ƒtkrHˆj  ¡ D ]\}}ˆd|› �|ƒ q.d S )NÚf_backÚf_codeÚ
f_builtinsÚ	f_globalsÚf_traceÚf_localszlocal )rl   r3   ÚdictrU   )rl   r8   Úlocal)rA   r;   r9   r   r   Úadd_frame_references¢   s
    z2annotated_references.<locals>.add_frame_referencesc                      s   ˆ dddƒ d S )NÚ__objclass__rI   Ú__doc__r   r   rD   r   r   Ú add_getset_descriptor_references¬   s    z>annotated_references.<locals>.add_getset_descriptor_referencesÚ__dict__Ú	__class__Ú__mro__)ÚtupleÚlistrm   ÚsetÚ	frozensetÚtypesÚFunctionTypeÚ	FrameTypeÚCellTypeÚ
MethodTypera   rb   ÚGetSetDescriptorTyper3   ru   Ú
isinstance)r9   rE   rM   rS   rX   r[   r^   rf   ro   rr   Ztype_based_referencesÚtype_r   )rA   r;   r9   r:   r   Úannotated_references^   sT    	


           õ
r‚   é    c                 C   sŽ  dd„ }t | tƒrt| ƒS t| ƒjdkr4d| j› �S t | tjƒrpz| jj}W n t	k
rd   d}Y nX d|› �S t | t
ƒrŠd|| ƒ› d�S t | tƒr¤d	|| ƒ› d
�S t | tƒr¾dt| ƒ› d�S t | tjƒrÖd| j› �S t | tƒ� rîd| j› �S t | tjƒ�r"| ƒ }|dk�rdS dt|ƒd›�S nht | tjƒ�rn| jj}t|ƒtk�r\d|td  d…  }d|› d| j› �S dt| ƒj› dt| ƒj› �S dS )zy
    Return a string to be used for Graphviz nodes.  The string
    should be short but as informative as possible.

    c                 S   sD   d  dd„ ttdƒ| ƒD ƒ¡}t| ƒdkr@|› dt| ƒd › �}|S )Nú,c                 s   s.   | ]&\}}t |tƒrt|ƒnt|ƒjV  qd S r'   )r€   Ú
BASE_TYPESrV   r3   rI   )Ú.0Úir/   r   r   r   Ú	<genexpr>Ö   s     z=object_annotation.<locals>.format_sequence.<locals>.<genexpr>é   z, ...)ÚjoinÚzipÚrangerd   )r9   Úbodyr   r   r   Úformat_sequenceÕ   s    z*object_annotation.<locals>.format_sequenceÚfunctionz	function
z<anonymous>zinstancemethod
rN   rO   ú(ú)zdict[zmodule
ztype
Nzweakref (dead referent)zweakref to id 0xr/   z...é   zframe
ú:zobject
Ú.)r€   r…   rV   r3   rI   rz   r~   r]   Ú__qualname__ÚAttributeErrorrw   rv   rm   rd   Ú
ModuleTypera   rb   r7   r|   rh   Úco_filenameÚFRAME_FILENAME_LIMITÚf_linenorJ   )r9   rŽ   Ú	func_nameZreferentÚfilenamer   r   r   Úobject_annotationÎ   s@    






r�   c                   @   s>   e Zd ZU eed< ee ed< eed< eeee	f  ed< dS )ÚNodeÚlabelÚcontextÚrootÚ
referrentsN)
rI   rJ   r•   ÚstrrK   r   Úboolr   r   Úintr   r   r   r   rž      s   
rž   ©r    Úfilterc                   sš  ˆ d krt ƒ ‰ ˆd krt‰‡ ‡fdd„| D ƒ}dd„ | D ƒ}dd„ t| ƒD ƒ}| D ]Œ}|t|ƒ }|| }t|ƒ}	t |¡D ]`}
t|
ƒ}| |d ¡}|d kržq||| }|	 |dg¡}||  |¡ |D ]}|j	 ||f¡ qÆq|qRdd„ t|ƒD ƒ}t
ƒ }|�r0| ¡ }||k�rqø| |¡ || }| |¡ qøi ‰g }t|ƒD ]*\}}||k�r@tˆƒˆ|< | |¡ �q@|D ]$}‡fdd„|j	D ƒ|j	d d …< �qp|S )	Nc                    s&   g | ]}t t|ƒˆ |ƒˆ|ƒg ƒ‘qS r   )rž   r�   ©r†   r9   r¦   r   r   Ú
<listcomp>  s     z create_graph.<locals>.<listcomp>c                 S   s   g | ]}g ‘qS r   r   r¨   r   r   r   r©     s     c                 S   s   i | ]\}}t |ƒ|“qS r   )r7   )r†   r‡   r9   r   r   r   Ú
<dictcomp>  s      z create_graph.<locals>.<dictcomp>ú?c                 S   s   g | ]\}}|j r|‘qS r   )r¡   )r†   r‡   Únr   r   r   r©     s      c                    s$   g | ]\}}|ˆ kr|ˆ | f‘qS r   r   )r†   rŸ   Úidx)Úid_to_filtered_idr   r   r©   /  s   þ)Úcuda_allocation_contextÚis_cuda_tensorrP   r7   r‚   r   rc   Úgetr,   r¢   rx   ÚpopÚaddÚextendrd   )Úobjectsr    r§   ÚnodesZnode_referrersZ
id_to_noder9   Zfidxr2   r:   Z	referrentZridZtidxÚtÚlabelsrŸ   Z	to_searchZto_keepr­   Z	referrersÚfilteredr‡   r¬   r   )r    r§   r®   r   Úcreate_graph  sR    



ÿrº   c                 C   s
   t  | ¡S r'   )ÚjsonÚdumps)r¬   r   r   r   Úescape4  s    r½   c                 C   s   t | tjƒo| jS r'   )r€   r   ZTensorZis_cuda)r9   r   r   r   r°   8  s    r°   c                     sp   t jj ¡ } i ‰ | d D ]F}|d }|d D ]0}|d dkrPt|ƒ\}}|ˆ |< ||d 7 }q,q‡ fdd„}|S )	NÚsegmentsÚaddressÚblocksÚstateZactive_allocatedÚsizec                    s<   t | ƒr8|  ¡  ¡ }ˆ  |¡}|d k	r8d t|dd�¡S d S )NÚ
T)Zfull_filename)r°   Zuntyped_storageZdata_ptrr±   rŠ   r	   )r9   ÚaddrÚframes©Zaddr_to_framer   r   Úobject_contextF  s    
z/cuda_allocation_context.<locals>.object_context)r   r   ÚmemoryZ	_snapshotr
   )ZsnapshotÚsegrÄ   ZblkrÅ   Z	real_sizerÇ   r   rÆ   r   r¯   ;  s    r¯   c              
   C   sž   dddg}t | ƒD ]4\}}| |› dt|jƒ› d|jr:dnd› d�¡ qt | ƒD ]8\}}|jD ](\}}| |› d	|› d
t|ƒ› d�¡ q^qP| d¡ d |¡S )Nzdigraph GraphName {znode [shape=rect];zrankdir=LR;z [label=z, color=ÚredÚblackz];z -> z
 [label = rO   z}
rÃ   )rP   r,   r½   rŸ   r¡   r¢   rŠ   )r¶   Úlinesr‡   r¬   r2   rŸ   Újr   r   r   Úto_dotO  s    
.$
rÎ   az  
<!DOCTYPE html>
<html>
<head>
  <style>
    body {
      margin: 0;
      padding: 0;
      overflow: hidden;
    }

    #container {
      display: flex;
      flex-direction: column;
      height: 100vh;
    }

    #main {
      flex: 2;
      overflow: auto;
    }

    #preContainer {
      flex: 1;
      overflow: auto;
    }

    svg {
        overflow: scroll;
    }

    pre {
      margin: 0;
      padding: 10px;
    }
  </style>
</head>
<body>
  <div id="container">
    <div id="main">
    </div>
    <div id="preContainer">
      <pre id="stacktrace">Mouse over tensor objects to see where they were allocated.</pre>
    </div>
  </div>
<script src='https://cdnjs.cloudflare.com/ajax/libs/viz.js/1.8.0/viz-lite.js'></script>
<script>
let dot = $DOT
let image = Viz(dot, {format: 'svg'});
document.getElementById('main').innerHTML = image
$LISTENERS
</script>
</body>
</html>
z˜
document.getElementById('node{id}').addEventListener('mouseover', function(event) {{
  document.getElementById("stacktrace").textContent = {stack}
}})
c                 C   sz   g }t | ƒD ]F\}}|jd kr qtjt|d ƒt|j› d|j› �ƒd�}| |¡ qt| ƒ}t	 
dt|ƒ¡ 
dd |¡¡S )Nr_   z:
)r7   Ústackz$DOTz
$LISTENERSrÃ   )rP   r    Ú_listener_templateÚformatr£   r½   rŸ   r,   rÎ   Ú	_templateÚreplacerV   rŠ   )r¶   Z	listenersr‡   r¬   ÚsÚdotr   r   r   Úto_html–  s    
(rÖ   c                    s$   t jjjdd� ‡ fdd„}t|ƒS )Ni † )Zmax_entriesc                    s8   | r4t dd„ | D ƒƒs$t d¡ d S ˆ tt| ƒƒƒ d S )Nc                 s   s   | ]}t |ƒV  qd S r'   )r°   r¨   r   r   r   rˆ   ¥  s     z:observe_tensor_cycles.<locals>.observer.<locals>.<genexpr>z No CUDA Tensors found in garbage)Úanyr   r    rÖ   rº   )r   ©Úcallbackr   r   r!   £  s
    
z'observe_tensor_cycles.<locals>.observer)r   r   rÈ   Z_record_memory_historyr-   )rÙ   r!   r   rØ   r   Úobserve_tensor_cycles   s    rÚ   c                  C   s   t  d¡ dd„ } t| ƒS )a˜  
    Reference cycles are freed by the cycle collector rather than being cleaned up
    when the objects in the cycle first become unreachable. If a cycle points to a tensor,
    the CUDA memory for that tensor will not be freed until garbage collection runs.
    Accumulation of CUDA allocations can lead to out of memory errors (OOMs), as well as
    non-deterministic allocation behavior which is harder to debug.

    This function installs a warning that is reports whenever a cycle that is holding CUDA
    memory is observed. The warning produces a html file that visualizes the cycle,
    and links it to the stack frame that allocted the CUDA tensor.
    z2Watching Python reference cycles for CUDA Tensors.c              	   S   s6   t dddd��}| | ¡ t d|j¡ W 5 Q R X d S )NÚwz.htmlF)ÚsuffixÚdeletezDReference cycle includes a CUDA Tensor see visualization of cycle %s)r   Úwriter   r   r8   )Úhtmlr2   r   r   r   Úwrite_and_logº  s    
z)warn_tensor_cycles.<locals>.write_and_log)r   r    rÚ   )rà   r   r   r   Úwarn_tensor_cycles¬  s    
rá   )/r   r   Útypingr   r   r   r   r   r   rz   ra   r»   Útempfiler   r   Ztorch.cuda._memory_vizr	   r
   r*   ÚloggingÚ	getLoggerrI   r   r-   r5   r}   r‚   r¥   ÚfloatÚcomplexr3   r£   Úbytesr…   r™   r�   rž   rº   r½   r°   r¯   rÎ   rÒ   rÐ   rÖ   rÚ   rá   r   r   r   r   Ú<module>   s:    
Im2.7
