U
    ªÊ¹jðó ã                   @   sª  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZ d dlmZ d dlZd dlmZmZ d dlZ d dl!Z d dl"m#  m$Z% d d	l&m'Z' d d
l(m)Z)m*Z*m+Z+m,Z,m-Z- d dl.m/Z/ d dl0m1Z1m2Z2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8 ddl9m:Z: ddl6m;Z;m<Z< ddl#m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZH ddlImJZJmKZK e LeM¡ZNejejOdd�ZOe jJjPZPdd„ ZQdd„ ZRdd„ ZSdd„ ZTdd„ ZUd d d!dgZVd"d#„ ZWeeX d$œd%d&„ZYdôd(d)„ZZG d*d+„ d+ƒZ[G d,d-„ d-e[ƒZ\G d.d/„ d/e[ƒZ]G d0d1„ d1e[ƒZ^G d2d3„ d3e[ƒZ_G d4d5„ d5e[ƒZ`d6e_iZad7d8„ Zbd9d:„ Zcd;d<„ Zdd=d>„ ZeejfG d?d@„ d@ƒƒZgejfG dAdB„ dBegƒƒZhdCdD„ ZiG dEdF„ dFehƒZjejfG dGdH„ dHejƒƒZkG dIdJ„ dJeƒZlG dKdL„ dLeƒZmeRdMƒeRdNƒeRdOƒeRdPƒeRdQƒeRdRƒdSœZndTdU„ ZoejfG dVdW„ dWehƒƒZpdXdY„ ZqG dZd[„ d[epƒZrd\d]„ Zsd^d_„ Ztdõdadb„Zuejeud'dc�Zvddde„ ZwejfG dfdg„ dgegƒƒZxejfG dhdi„ diexƒƒZyejfG djdk„ dkexƒƒZzG dldm„ dmexƒZ{ejfG dndo„ doexƒƒZ|ejfG dpdq„ dqe|ƒƒZ}ejfG drds„ dsexƒƒZ~G dtdu„ due}ƒZG dvdw„ dwegƒZ€ejfG dxdy„ dye€ƒƒZ�ejfG dzd{„ d{e€ƒƒZ‚ejfG d|d}„ d}egƒƒZƒG d~d„ deƒƒZ„G d€d�„ d�eƒƒZ…G d‚dƒ„ dƒeƒƒZ†G d„d…„ d…eƒƒZ‡ejfG d†d‡„ d‡egƒƒZˆG dˆd‰„ d‰eˆƒZ‰G dŠd‹„ d‹e‰ƒZŠG dŒd�„ d�egƒZ‹G dŽd�„ d�egƒZŒejfG d�d‘„ d‘eˆƒƒZ�G d’d“„ d“eˆƒZŽejfG d”d•„ d•eˆƒƒZ�G d–d—„ d—e�ƒZ�G d˜d™„ d™e�ƒZ‘ejfG dšd›„ d›e�ƒƒZ’ejfG dœd�„ d�e’ƒƒZ“G dždŸ„ dŸe“ƒZ”G d d¡„ d¡e’ƒZ•G d¢d£„ d£e’ƒZ–G d¤d¥„ d¥e’ƒZ—G d¦d§„ d§e’ƒZ˜G d¨d©„ d©e“ƒZ™G dªd«„ d«egƒZšejfG d¬d­„ d­e•ƒƒZ›ejfG d®d¯„ d¯egƒƒZœG d°d±„ d±e’ƒZ�död²d²d²eeX eeX eeX eXežeeX d³œ	d´dµ„ZŸd²d²d²d¶œd·d¸„Z G d¹dº„ dºe•ƒZ¡G d»d¼„ d¼e•ƒZ¢G d½d¾„ d¾e•ƒZ£G d¿dÀ„ dÀe•ƒZ¤G dÁdÂ„ dÂe•ƒZ¥G dÃdÄ„ dÄe•ƒZ¦G dÅdÆ„ dÆe•ƒZ§G dÇdÈ„ dÈe•ƒZ¨G dÉdÊ„ dÊe•ƒZ©G dËdÌ„ dÌe•ƒZªG dÍdÎ„ dÎe•ƒZ«ejfG dÏdÐ„ dÐegƒƒZ¬G dÑd²„ d²e¬ƒZ­G dÒdÓ„ dÓe¬ƒZ®G dÔdÕ„ dÕe j¯j°ƒZ±G dÖd×„ d×ƒZ²G dØdÙ„ dÙƒZ³G dÚdÛ„ dÛe•ƒZ´G dÜdÝ„ dÝe’ƒZµG dÞdß„ dßeµƒZ¶G dàdá„ dáeµƒZ·G dâdã„ dãe’ƒZ¸G dädå„ dåe’ƒZ¹G dædç„ dçe�ƒZºG dèdé„ dée¶ƒZ»G dêdë„ dëe¶ƒZ¼G dìdí„ díe·ƒZ½G dîdï„ dïe·ƒZ¾G dðdñ„ dñe·ƒZ¿G dòdó„ dóe·ƒZÀdS )÷é    N)Únullcontext)ÚEnum)Úpartial)Ú	signature)
ÚAnyÚCallableÚClassVarÚDictÚListÚOptionalÚSequenceÚSetÚTupleÚUnion)Úpatch)ÚExprÚInteger)Úidentity)Úcompute_required_storage_lengthÚis_boolean_dtypeÚis_float_dtypeÚmake_channels_last_strides_forÚmake_contiguous_strides_for)Úget_signature_for_torch_op)ÚCleanDivÚFloorDivÚModularIndexingé   )ÚconfigÚdependencies)Úindex_prevent_reordering)Úget_device_properties)Úextract_read_writesÚvar_builder)ÚargsortÚcache_on_selfÚconvert_shape_to_inductorÚconvert_shape_to_symintÚdeveloper_warningÚget_kernel_metadataÚpad_listlikeÚ	sympy_dotÚsympy_productÚ
sympy_subsÚsympy_symbolÚtry_find_schema)ÚopsÚVz  ©Úprefixc                    s   ‡ fdd„‰ ˆ | ƒ d S )Nc              	      sf   t | ttfƒr"| D ]}ˆ |ƒ qn@t | tjjjttt	j
t	jjjttjjjfƒsbtdt| ƒ› d�ƒ‚d S )NzFound zE, which is not a supported top level IR node. See [Note: Inductor IR])Ú
isinstanceÚlistÚtupleÚtorchZ	_inductorZirÚ
ExpandViewÚDynamicScalarÚ	TensorBoxÚsympyÚSymbolÚlogicÚboolalgÚBooleanr   ÚAssertionErrorÚtype)ÚnodesÚnode©Ú_check_tensorbox© úQ/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/_inductor/ir.pyrE   i   s     ùþõz%validate_ir.<locals>._check_tensorboxrF   )Znode_or_nodesrF   rD   rG   Úvalidate_irh   s    rH   c                    s   t ˆ tƒst‚‡ fdd„}|S )Nc                     s   t tˆ ƒ| |ŽS ©N)Úgetattrr0   )ÚargsÚkwargs©ÚnamerF   rG   Úfn„   s    zops_wrapper.<locals>.fn)r4   Ústrr@   )rN   rO   rF   rM   rG   Úops_wrapper�   s    rQ   c                    s&   t t| tt| ƒƒƒƒ‰ ‡ fdd„}|S )Nc                    s0   t ˆ ƒt ˆƒkst‚‡ ‡fdd„tt ˆ ƒƒD ƒS )Nc                    s   g | ]}ˆ ˆ|  ‘qS rF   rF   ©Ú.0Úi)ÚindexÚ	inv_orderrF   rG   Ú
<listcomp>�   s     z4inverse_reorder.<locals>.reindex.<locals>.<listcomp>©Úlenr@   Úrange©rU   ©rV   r[   rG   Úreindex�   s    z inverse_reorder.<locals>.reindex)ÚdictÚziprZ   rY   ©Úorderr]   rF   r\   rG   Úinverse_reorderŠ   s    rb   c                    s   ‡ fdd„}|S )Nc                    s0   t ˆ ƒt ˆƒkst‚‡ ‡fdd„tt ˆ ƒƒD ƒS )Nc                    s   g | ]}ˆ ˆ|  ‘qS rF   rF   rR   )rU   ra   rF   rG   rW   —   s     z1same_reorder.<locals>.reindex.<locals>.<listcomp>rX   r[   ©ra   r[   rG   r]   •   s    zsame_reorder.<locals>.reindexrF   r`   rF   rc   rG   Úsame_reorder”   s    rd   c                    s   ‡ ‡fdd„}|S )Nc                    s   ˆ ˆ| ƒƒS rI   rF   r[   ©Úreindex1Úreindex2rF   rG   r]   �   s    z fuse_reindexing.<locals>.reindexrF   )rf   rg   r]   rF   re   rG   Úfuse_reindexingœ   s    rh   é   é   c                    s0   dd„ t | ƒD ƒ‰ ‡ fdd„tt| ƒƒD ƒ}|S )z‡
    Convert stride order to fill order
    For channel last format,
    stride order = [3, 0, 2, 1] and fill order = [1, 3, 2, 0]
    c                 S   s   i | ]\}}||“qS rF   rF   ©rS   ÚidxÚposrF   rF   rG   Ú
<dictcomp>¬   s      z+stride_order2fill_order.<locals>.<dictcomp>c                    s   g | ]}ˆ | ‘qS rF   rF   rR   ©ÚlookuprF   rG   rW   ­   s     z+stride_order2fill_order.<locals>.<listcomp>)Ú	enumeraterZ   rY   )ra   Ú
fill_orderrF   ro   rG   Ústride_order2fill_order¦   s    rs   )Úseqc                 C   s<   t | ƒ}dd„ tt| ƒƒD ƒ}t|ƒD ]\}}|||< q&|S )z)
    Convert strides to stride order
    c                 S   s   g | ]}d ‘qS rI   rF   ©rS   Ú_rF   rF   rG   rW   ¶   s     z$get_stride_order.<locals>.<listcomp>)r$   rZ   rY   rq   )rt   Z
sorted_idxÚoutrT   ÚelemrF   rF   rG   Úget_stride_order±   s
    
ry   Tc                    sš   | d krd S |st jjj‰ nt‰ ‡ fdd„|  ¡ D ƒ}t| ƒrX‡ fdd„|  ¡ jD ƒ}nt	|ƒ}|  
¡ }|  ¡ }t|ƒ}t|ƒ}tj||||d� ¡ }|S )Nc                    s   g | ]}ˆ |ƒ‘qS rF   rF   ©rS   Ús©Zshape_fnrF   rG   rW   Ä   s     z%ir_node_to_tensor.<locals>.<listcomp>c                    s   g | ]}ˆ |ƒ‘qS rF   rF   rz   r|   rF   rG   rW   Æ   s     )ÚsizeÚstrideÚdtypeÚdevice)r1   ÚgraphÚsizevarsÚ	size_hintr   Úget_sizeÚis_storage_and_layoutÚ
get_layoutr~   r   Ú	get_dtypeÚ
get_devicer'   r7   Zempty_stridedZzero_)ÚxÚguard_shaper}   r~   r   r€   ÚtrF   r|   rG   Úir_node_to_tensor¼   s(       ÿ
rŒ   c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚOptionalAttrc                 C   s
   d| _ d S )NZoptional_attrrM   ©ÚselfrF   rF   rG   Ú__init__Ô   s    zOptionalAttr.__init__c                 C   s   | j S rI   rM   rŽ   rF   rF   rG   Ú__repr__×   s    zOptionalAttr.__repr__N)Ú__name__Ú
__module__Ú__qualname__r�   r‘   rF   rF   rF   rG   r�   Ó   s   r�   c                   @   s   e Zd Zdd„ ZdS )ÚOptionalStringc                 C   s
   d| _ d S )NÚoptional_stringrM   rŽ   rF   rF   rG   r�   Ü   s    zOptionalString.__init__N©r’   r“   r”   r�   rF   rF   rF   rG   r•   Û   s   r•   c                   @   s   e Zd Zdd„ ZdS )ÚOptionalListc                 C   s
   d| _ d S )NÚoptional_listrM   rŽ   rF   rF   rG   r�   á   s    zOptionalList.__init__Nr—   rF   rF   rF   rG   r˜   à   s   r˜   c                   @   s   e Zd Zdd„ ZdS )ÚOptionalScalarc                 C   s
   d| _ d S )NÚoptional_scalarrM   rŽ   rF   rF   rG   r�   æ   s    zOptionalScalar.__init__Nr—   rF   rF   rF   rG   rš   å   s   rš   c                   @   s   e Zd Zdd„ ZdS )ÚOptionalLayoutc                 C   s
   d| _ d S )NZoptional_layoutrM   rŽ   rF   rF   rG   r�   ë   s    zOptionalLayout.__init__Nr—   rF   rF   rF   rG   rœ   ê   s   rœ   c                   @   s   e Zd Zdd„ ZdS )ÚOptionalTensorc                 C   s
   d| _ d S )NZoptional_tensorrM   rŽ   rF   rF   rG   r�   ð   s    zOptionalTensor.__init__Nr—   rF   rF   rF   rG   r�   ï   s   r�   zOptional[Layout]c                 C   s   |st jjr| S |S rI   )r1   r�   Úcpp_wrapper)Zoptional_valueÚvaluerF   rF   rG   Úmay_convert_to_optional÷   s    r    c                 C   s.   t | dd ƒrt|  ¡ ƒS t| tjƒr*| jS d S )Nrˆ   )rJ   Úget_device_typerˆ   r4   r7   r€   rA   ©r‰   rF   rF   rG   r¡   û   s
    r¡   c                 C   s   t | ƒdkS )NÚcuda©r¡   r¢   rF   rF   rG   Ú	is_triton  s    r¥   c                 C   s   t | ƒdkS )NÚcpur¤   r¢   rF   rF   rG   Úis_cpu  s    r§   c                   @   s�   e Zd ZU eƒ Zeee  ed< e	e
jeejj dœdd„ƒƒZdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Zedd„ ƒZdd„ Zdd„ Zdd„ ZdS )ÚIRNodeÚ_current_origins)Úoriginsc                 c   s(   t j}|| B t _z
d V  W 5 |t _X d S rI   )r¨   r©   )rª   ÚoldrF   rF   rG   Úcurrent_origins  s
    

zIRNode.current_originsc                 C   s$   t | jƒ| _tjrt ¡ nd | _d S rI   )Úsetr©   rª   r   Údebug_ir_tracebackÚ	tracebackÚformat_stackrŽ   rF   rF   rG   Ú__post_init__  s    zIRNode.__post_init__c                 C   s   | j S rI   )r¯   rŽ   rF   rF   rG   Úget_traceback  s    zIRNode.get_tracebackc                 C   s6   dt | ddƒ› �}t|ƒdkr0|d d… › d�}|gS )Nzorigins=rª   Ú é@   é=   z...)rJ   rY   )r�   rª   rF   rF   rG   Úcommon_repr   s    zIRNode.common_reprc                 C   s6   ||   ¡  }td tt|ƒ¡ƒ}t| ƒj› d|› d�S )Nz,
z(
z
))r¶   ÚindentÚjoinÚmaprP   rA   r’   ©r�   ÚlinesrF   rF   rG   Ú
str_helper'  s    zIRNode.str_helperc                 C   s   ||   ¡ kS rI   )Úget_read_names©r�   rN   rF   rF   rG   Ú
is_user_of,  s    zIRNode.is_user_ofc                 C   s   dd„ |   ¡ D ƒS )Nc                 S   s   h | ]
}|j ’qS rF   rM   )rS   ÚdeprF   rF   rG   Ú	<setcomp>1  s     z(IRNode.get_read_names.<locals>.<setcomp>)Ú	get_readsrŽ   rF   rF   rG   r½   /  s    zIRNode.get_read_namesc                 C   s   t |  ¡ ƒS rI   )r,   r„   rŽ   rF   rF   rG   Ú	get_numel3  s    zIRNode.get_numelc                 C   s   t jj t |  ¡ d¡¡S ©Nr   ©r1   r�   r‚   Úis_expr_static_and_truer;   ÚEqrÃ   rŽ   rF   rF   rG   Úis_zero_elements6  s    zIRNode.is_zero_elementsc                 C   s   t dt| ƒ› �ƒ‚dS )a)  
        If the IRNode refers to data which has not been materialized (e.g.,
        it is a Pointwise/Reduction that could potentially have more
        compute fused into it), realize the IRNode into physical memory,
        ending the possibility of fusing into it, but allowing, e.g., multiple
        users to access the data without having to recompute.

        Check StorageBox.realize for a particularly notable implementation.

        TODO(ezyang): I think, in principle, every IRNode should have an
        implementation of this, and most of the time no-op is OK, but you
        really do have to audit each IRNode for this, so for now, raise
        an error if it's not implemented.  Note that some code in graph.py
        will catch this thrown error and suppress it with a warning.
        zrealize NYI on N)ÚNotImplementedErrorrA   rŽ   rF   rF   rG   Úrealize9  s    zIRNode.realizeN)r’   r“   r”   r­   r©   r   r   r   Ú__annotations__ÚstaticmethodÚ
contextlibÚcontextmanagerr7   ÚfxÚNoder¬   r±   r²   r¶   r¼   r¿   r%   r½   rÃ   rÈ   rÊ   rF   rF   rF   rG   r¨     s   

r¨   c                       sº   e Zd ZU ejed< ejed< edef ed< e	e
 ed< d dd„Z‡ fd	d
„ZeZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zedd„ ƒZed!dd„ƒZedd„ ƒZdd„ Zdd„ Z‡  ZS )"ÚLoopsr€   r   .Úinner_fnÚranges©rÓ   c                    sF   ˆ   dˆ jj› d�tˆ jƒˆ  ¡ g‡ fdd„|D ƒ dˆ j›�g ¡S )Nú'c                    s    g | ]}|› d t ˆ |ƒ› �‘qS ©ú=)rJ   )rS   rN   rŽ   rF   rG   rW   Z  s     z!Loops.__str__.<locals>.<listcomp>úorigin_node=)r¼   r€   rA   rP   r   Úinner_fn_strÚorigin_node)r�   ÚnamesrF   rŽ   rG   Ú__str__S  s    ýûúÿzLoops.__str__c                    s   t ƒ  ¡  d | _d S rI   ©Úsuperr±   rÚ   rŽ   ©Ú	__class__rF   rG   r±   ^  s    
zLoops.__post_init__c                 C   s   | j S rI   ©r   rŽ   rF   rF   rG   r‡   d  s    zLoops.get_dtypec                 C   s   | j S rI   ©r€   rŽ   rF   rF   rG   rˆ   g  s    zLoops.get_devicec                 C   s   | j S rI   ©rÚ   rŽ   rF   rF   rG   Úget_origin_nodej  s    zLoops.get_origin_nodec                 C   s   | j S rI   rÔ   rŽ   rF   rF   rG   r„   m  s    zLoops.get_sizec                 C   s   dS ©NFrF   rŽ   rF   rF   rG   Ú	is_externp  s    zLoops.is_externc                 O   sJ   |  dd ¡}|  dd ¡}| ||Ž}||_tjr:|p<t ¡ nd |_t |¡S )NrÚ   r¯   )ÚpoprÚ   r   r®   r¯   r°   r:   Úcreate)ÚclsrK   rL   rÚ   ÚtbÚrrF   rF   rG   rè   s  s    
ÿzLoops.createrT   c                    s   ‡ fdd„t | ƒD ƒS )Nc                    s2   g | ]*\}}|d krt  d¡ntˆ › |› �ƒ‘qS )r   r   )r;   r   r.   )rS   Únr{   r2   rF   rG   rW   €  s   ÿz Loops._index.<locals>.<listcomp>©rq   )rÓ   r3   rF   r2   rG   Ú_index~  s    
þzLoops._indexc                 C   s   t |  ¡ ƒS rI   )rY   rÙ   rŽ   rF   rF   rG   Úinner_fn_str_len…  s    zLoops.inner_fn_str_lenc                 C   s   |   | j¡}tj | j|¡S rI   )rî   rÓ   r1   ÚKernelFormatterHandlerÚir_to_stringrÒ   )r�   rU   rF   rF   rG   rÙ   ‰  s    zLoops.inner_fn_strc              
   C   sl   t  tdd¡�T |  ¡ r>t|  ¡ |  ¡ |  ¡ ƒjW  5 Q R £ S t|  ¡ |  ¡ ƒjW  5 Q R £ S W 5 Q R X d S ©NÚallow_indexingT)	r   ÚobjectÚFlexibleLayoutÚget_reduction_typer"   Úmake_loaderr„   Úget_reduction_sizeÚreadsrŽ   rF   rF   rG   rÂ   �  s    ýþzLoops.get_reads)rÔ   )rT   )r’   r“   r”   r7   r€   rË   r   r   r   r
   r   rÜ   r±   r‘   r‡   rˆ   rä   r„   ræ   Úclassmethodrè   rÌ   rî   r%   rï   rÙ   rÂ   Ú__classcell__rF   rF   rß   rG   rÑ   L  s(   






rÑ   c                C   s&   |j rt tdƒ|¡S t d|¡S d S )NÚnanr   )Úis_floating_pointr0   ÚconstantÚfloat)rl   r   rF   rF   rG   Únop_loader_fnœ  s    r   c                   @   s4   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ ZdS )Ú	Pointwisec                 C   s   |   ¡ rtt| jd�S | jS )Nrá   )rÈ   r   r   r   rÒ   rŽ   rF   rF   rG   r÷   ¤  s    zPointwise.make_loaderc                 C   s   g S rI   rF   rŽ   rF   rF   rG   rø   «  s    zPointwise.get_reduction_sizec                 C   s   d S rI   rF   rŽ   rF   rF   rG   rö   ®  s    zPointwise.get_reduction_typec                 C   s   |   ¡ }t |||ƒ||ƒ¡S rI   )r÷   r0   Ústore©r�   Úoutput_nameÚindexerÚvarsÚloaderrF   rF   rG   Ústore_output±  s    zPointwise.store_outputc                 C   s,   |   ¡ }t td|¡|ƒ}t|| j|| jƒS ©úFMove this to a given device. Requires that all reads are to constants.Úoverride_device)r÷   r   rô   ÚConstantBufferr  r   rÓ   ©r�   r€   r  rF   rF   rG   Úconstant_to_deviceµ  s    zPointwise.constant_to_deviceN)r’   r“   r”   r÷   rø   rö   r  r  rF   rF   rF   rG   r  £  s
   r  c                   @   sD   e Zd ZU eee gef ed< dZee	 ed< dd„ Z
dd„ ZdS )ÚScatterÚoutput_indexerNÚscatter_modec                 C   s4   |   ¡ }t td|¡|ƒ}t|| j|| j| j| jƒS r	  )	r÷   r   rô   r  r  r   rÓ   r  r  r  rF   rF   rG   r  Á  s    úzScatter.constant_to_devicec                 C   s*   |   ¡ }tj|||  |¡ƒ||ƒ| jd�S )N)Úmode)r÷   r0   r  r  r  r  rF   rF   rG   r  Î  s    üzScatter.store_output)r’   r“   r”   r   r
   r   rË   r  r   rP   r  r  rF   rF   rF   rG   r  ¼  s   
r  c                   @   s   e Zd ZdZdZdZdZdS )ÚReductionHintr   r   rj   ri   N)r’   r“   r”   ÚINNERÚOUTERÚ
OUTER_TINYÚDEFAULTrF   rF   rF   rG   r  Ø  s   r  c                   @   s   e Zd ZdZdZdS )ÚTileHintr   r   N)r’   r“   r”   ÚSQUAREr  rF   rF   rF   rG   r  ß  s   r  Ú
logical_orÚmaximumÚminimumÚmulÚaddZbitwise_xor)ÚanyÚmaxÚminÚprodÚsumÚxor_sumc                    sN   ˆt krt ˆ }n8ˆdkr*‡ ‡fdd„}n ˆdkr<dd„ }ntdˆ› �ƒ‚|S )N¾   ÚargmaxÚargminc           
   
      s¾   | \}}|\}}ˆdkr&t  ||¡}nt  ||¡}t  ||¡}tˆ ƒr†t  ||¡}t  ||¡}	t  |t  ||	¡¡}t  |t  ||	¡¡}t  |t  |t  ||¡¡¡}t  |||¡t  |||¡fS )Nr'  )	r0   ÚltÚgtÚeqr   Úner  Úlogical_andÚwhere)
ÚaÚbZa_valueZa_indexZb_valueZb_indexÚmaskÚequalZa_isnanZb_isnan©r   Úreduction_typerF   rG   Ú
combine_fnó  s$     ÿþz,get_reduction_combine_fn.<locals>.combine_fnÚwelford_combinec                 S   sR   | \}}}|\}}}|| }|| }	||	 }
|||
  || || | |
  |	fS rI   rF   )r.  r/  Za_meanZa_m2Za_weightZb_meanZb_m2Zb_weightÚdeltaZ
new_weightZ	w2_over_wrF   rF   rG   r4    s    


ýzunknown reduction_type=)ÚREDUCTION_COMBINE_FNrÉ   )r3  r   r4  rF   r2  rG   Úget_reduction_combine_fnî  s    

r8  c                   @   s<  e Zd ZU ee ed< eed< ejed< e	ed< dd„ Z
dd„ Zd	d
„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zedd„ ƒZedd„ ƒZee	jfejejejedef ee ee ee	dœdd„ƒZedd„ ƒZedd „ ƒZeeee	e	d!œd"d#„ƒZed$d%„ ƒZeejejejedef ee ee eee	d&œ	d'd(„ƒZ d)S )*Ú	ReductionÚreduction_rangesr3  Ú	src_dtypeÚreduction_hintc                 C   s   t j| dd�S )N)rÓ   r:  r3  )rÛ   )rÑ   rÜ   rŽ   rF   rF   rG   rÜ   (  s     ÿzReduction.__str__c                 C   s   |   ¡ S rI   )rÜ   rŽ   rF   rF   rG   r‘   -  s    zReduction.__repr__c                 C   s   | j S rI   )r:  rŽ   rF   rF   rG   rø   0  s    zReduction.get_reduction_sizec                 C   s   | j S rI   ©r3  rŽ   rF   rF   rG   rö   3  s    zReduction.get_reduction_typec              	   C   s0   t  | j| j| j|  ||¡¡}t  |||ƒ|¡S rI   )r0   Ú	reductionr   r;  r3  rÒ   Ústore_reduction)r�   r  r  r  Úreduction_varsrŸ   rF   rF   rG   r?  6  s    
üzReduction.store_reductionc                 C   s   t | jƒt | jƒ S rI   )rY   rÓ   r:  rŽ   rF   rF   rG   Úindex_length?  s    zReduction.index_lengthc                 C   s,   |   | j¡}|   | jd¡}tj | j||¡S )Nrë   )rî   rÓ   r:  r1   rð   rñ   rÒ   )r�   rU   ÚrindexrF   rF   rG   rÙ   B  s    ýzReduction.inner_fn_strc              	   C   s<   |   ¡ }t td|¡|ƒ}t|| j|| j| j| j| j	t
jƒS r	  )r÷   r   rô   r  r9  r   rÓ   r:  r3  r;  r  r  r  rF   rF   rG   r  K  s    øzReduction.constant_to_devicec              	      sô  dd„ }t jj |¡}	t jj t|ƒ¡}
t| ƒoL|dkoLtjoL||	ƒoL||
ƒ}|s\tj	dfS t
| ƒj‰d‰d‰d‰ˆˆ ˆ ‰ˆˆ ˆ ‰ ‡ ‡‡‡‡‡fdd	„}‡ ‡‡‡‡‡fd
d„}|
dkrÎtj||	|
ƒfS |	ˆksæ|
ˆd d krðtj	dfS t| ||||||tj	ƒ}dd„ }||ƒ\}}|�r.||ƒ\}}t|ƒdk�rFtj	dfS t | ¡ | ¡ ¡\\}}}d}d}|D ]V}t jj ||¡}t jj ||| ¡ ¡}tdd„ |D ƒƒ}|�rº|d7 }n|d7 }�qn||k�ràtj||	|
ƒfS tj||	|
ƒfS d S )Nc                 S   s   t | ttjfƒS rI   )r4   Úintr;   r   r¢   rF   rF   rG   Ú
_is_statice  s    z(Reduction.num_splits.<locals>._is_staticr%  r   é    é   i   c           	         s  d}d| }|dˆ krdS | dkr(dS | | ˆkr:ˆ}n¼| | ˆk r¾ˆˆ d|  }|| d | }| ||  d ||  ‰ t  | ¡}t|‡ fdd„d�}t|ˆ  ƒd	k r¸t|ˆƒ}qöˆ }n8t  | ¡}t|‡fd
d„d�}t|ˆ ƒdk rò|}nˆ}| ||  d ||  S )Né   rE  rj   r   i    c                    s   t | ˆ  ƒS rI   ©Úabsr¢   ©Ztmp_split_sizerF   rG   Ú<lambda>“  ó    zFReduction.num_splits.<locals>.inner_reduction_splits.<locals>.<lambda>©Úkeyé   c                    s   t | ˆ  ƒS rI   rH  r¢   ©Úmax_elements_per_threadrF   rG   rK  ›  rL  é2   ©r;   Údivisorsr!  rI  r   )	Úreduction_numel_hintÚ
numel_hintÚ	num_warpsÚnum_threadsÚ
split_sizeÚtarget_blocksZblocks_per_outputrT  Úclosest©Zmax_elements_per_devicerQ  Zmin_elements_per_deviceZmin_elements_per_threadZnum_smZthreads_per_smrJ  rG   Úinner_reduction_splits�  s6    þ

ÿz4Reduction.num_splits.<locals>.inner_reduction_splitsc                    s  d}|d }d}d}|| d | }| | ˆk r6ˆ}n¸| | ˆk r¶ˆˆ | }|| d | }| ||  d ||  ‰ t  | ¡}	t|	‡ fdd„d�}
tˆ |
 ƒd	k r°t|
ˆƒ}qîˆ }n8t  | ¡}	t|	‡fd
d„d�}
t|
ˆ ƒdk rê|
}nˆ}| ||  d ||  S )NrG  rE  é   é€   r   c                    s   t | ˆ  ƒS rI   rH  r¢   rJ  rF   rG   rK  ¶  rL  zFReduction.num_splits.<locals>.outer_reduction_splits.<locals>.<lambda>rM  é   c                    s   t | ˆ  ƒS rI   rH  r¢   rP  rF   rG   rK  ½  rL  rR  rS  )rU  rV  rW  rX  Zrvals_per_threadZxvals_per_blockZxblocksrY  rZ  rT  r[  r\  rJ  rG   Úouter_reduction_splits¥  s4    þ

ÿz4Reduction.num_splits.<locals>.outer_reduction_splitsrj   c                    sÂ   t d t|  ¡ |  ¡ |  ¡ d�| d�}| ¡ }dd„ |jD ƒ}g }d}t|jdd„ d�D ]b‰ t	‡ fd	d
„|D ƒƒrV| 
ˆ j¡ ˆ jtjjkrVtjjˆ j }|jj}| ¡  |jj|krVd}qV||fS )N©r€   r   r}   ©rN   ÚlayoutÚdatac                 S   s(   g | ] }t |tjƒrt |tjƒs|‘qS rF   )r4   r;   r   ÚNumber©rS   rë   rF   rF   rG   rW   ì  s    þzBReduction.num_splits.<locals>.get_read_indices.<locals>.<listcomp>Fc                 S   s   | j S rI   rM   r¢   rF   rF   rG   rK  ó  rL  z@Reduction.num_splits.<locals>.get_read_indices.<locals>.<lambda>rM  c                 3   s   | ]}|ˆ j jkV  qd S rI   )rU   Zfree_symbolsrg  ©ÚmdrF   rG   Ú	<genexpr>ô  s     zAReduction.num_splits.<locals>.get_read_indices.<locals>.<genexpr>T)ÚComputedBufferrõ   rˆ   r‡   r„   Úget_read_writesÚ
range_varsÚsortedrù   ÚallÚappendrU   rN   r1   r�   Úname_to_bufferrd  r~   Údecide_layout)rë   ÚcbÚread_writesrm  ÚindicesÚchangedÚbufZoriginal_striderF   rh  rG   Úget_read_indicesÞ  s2    ýù	þz.Reduction.num_splits.<locals>.get_read_indicesr   c                 s   s   | ]}|d kV  qdS )r   NrF   rz   rF   rF   rG   rj    s     z'Reduction.num_splits.<locals>.<genexpr>)r1   r�   r‚   Zsymbolic_hintr,   r¥   r   Zsplit_reductionsr  r  r!   Zmulti_processor_countr  r9  rY   r   Úindex_vars_squeezer„   rø   Úsimplify_with_rangesÚstride_hintsÚkeysro  r  )r€   Ú	dst_dtyper;  rÒ   rÓ   r:  r3  Úreduction_numelrD  rU  rV  Zshould_splitr]  ra  rë   rx  ru  rv  rv   r@  Z	num_outerZ	num_innerrT   ÚstridesÚouterrF   r\  rG   Ú
num_splitsZ  s�    ÿÿúø	ö

$$ ÿÿþ
ø 
 ÿ

 ÿ ÿzReduction.num_splitsc                    sn   dd„ ˆD ƒ‰t ||ƒ‰ ‡ ‡‡fdd„‰|dkrbtddˆt ˆ¡ƒ ¡ ‰‡‡fdd„‰‡fd	d
„S ˆ‰ˆS dS )z1Convert inner_fn from a reduction to an pointwisec                 S   s   g | ]}t jj |¡‘qS rF   )r1   r�   r‚   Zevaluate_static_shape©rS   r‰   rF   rF   rG   rW     s    z2Reduction._unroll_reduction_fn.<locals>.<listcomp>c                    s,   t  ˆ‡ ‡fdd„tjdd„ ˆD ƒŽ D ƒ¡S )Nc                 3   s   | ]}ˆˆ |ƒV  qd S rI   rF   )rS   rB  )rU   Úvalue_fnrF   rG   rj  (  s   ÿz=Reduction._unroll_reduction_fn.<locals>.fn.<locals>.<genexpr>c                 S   s   g | ]}t |ƒ‘qS rF   )rZ   r‚  rF   rF   rG   rW   +  s     z>Reduction._unroll_reduction_fn.<locals>.fn.<locals>.<listcomp>)Ú	functoolsÚreduceÚ	itertoolsÚproductr[   )r4  r:  rƒ  r[   rG   rO   %  s    ÿþþz*Reduction._unroll_reduction_fn.<locals>.fn©r'  r&  Nc                    s*   dd„ |D ƒ}ˆ| |ƒt  ˆ |ƒtj¡fS )Nc                 S   s   g | ]}t  |¡‘qS rF   )r;   ÚexpandrR   rF   rF   rG   rW   9  s     zDReduction._unroll_reduction_fn.<locals>.value_fn.<locals>.<listcomp>)r0   Ú
index_exprr7   Úint64©rU   rB  )Úflatten_indexrÒ   rF   rG   rƒ  8  s    þz0Reduction._unroll_reduction_fn.<locals>.value_fnc                    s   ˆ | ƒd S ©Nr   rF   r[   )rO   rF   rG   rK  ?  rL  z0Reduction._unroll_reduction_fn.<locals>.<lambda>)r8  ÚFixedLayoutrõ   Úcontiguous_stridesÚmake_indexer)rÒ   r:  r3  r;  rF   )r4  r�  rO   rÒ   r:  rƒ  rG   Ú_unroll_reduction_fn  s     ÿ
üzReduction._unroll_reduction_fn.)r€   r}  r;  rÒ   rÓ   r:  r3  r<  c	                    sz  t jj tˆƒ¡}	|	dkr„‡ fdd„}
|
dƒ|
dƒ|
dƒ|
dƒdœ‰ˆˆ ¡ ks^tˆ› d�ƒ‚‡ ‡‡fdd„}tj|||t	|ƒd	�S |	dkrÀˆd
kr¢‡ fdd„}n‡‡fdd„}t |ˆ ||¡S t
|	tjƒ�rt jj |	¡tjk �rt|ƒdk�rt |ˆ |  ˆˆˆ|¡|¡S |  |ˆ |ˆ|ˆˆ|	¡\}}|tjk�r:|}|dk�r^|  |ˆ |ˆ|ˆˆ||¡	S t t|ˆ ˆ|ˆˆ||ƒ¡S )Nr   c                    s(   ˆ t jkrt| ƒS ˆ jr t| ƒS t| ƒS rI   )r7   Úboolrý   rÿ   rC  ©Úval©r}  rF   rG   Úpy_cnstV  s    ÿ
ÿ
ûz!Reduction.create.<locals>.py_cnstr   )r#  r$  r"  r  z* not supported for zero-dimension tensors!c                    s   t  ˆˆ ˆ ¡S rI   ©r0   rþ   r[   )r}  r3  Úrtypes_to_initsrF   rG   Úconst_fnk  s    z"Reduction.create.<locals>.const_fn©r€   r   rÒ   rÓ   rˆ  c                    s   t  dˆ ¡S rÄ   r˜  r[   r–  rF   rG   rO   y  s    zReduction.create.<locals>.fnc                    s   dd„ ˆD ƒ}ˆ | |ƒS )Nc                 S   s   g | ]}t  d ¡‘qS ©r   ©r;   r   ru   rF   rF   rG   rW     s     z0Reduction.create.<locals>.fn.<locals>.<listcomp>rF   ©rU   Úreduction_index)rÒ   r:  rF   rG   rO   ~  s    )r1   r�   r‚   Úsimplifyr,   r|  r@   r  rè   r5   r4   r;   r   rƒ   r   Zunroll_reductions_thresholdr’  r�  r  r  Úcreate_multilayerr:   r9  )ré   r€   r}  r;  rÒ   rÓ   r:  r3  r<  r~  r—  rš  rO   ÚhintÚsplitrF   )r}  rÒ   r:  r3  r™  rG   rè   D  sž    
ü	
ÿþü
ÿÿþ
ü   ÿú
ø
÷øÿzReduction.createc                 C   sv   | dkr0t |ƒrtdƒS t|ƒr$dS t |¡jS | dkr`t |ƒrHtdƒS t|ƒrTdS t |¡jS dddddddœ|  S )	N>   r&  r   z-infr   >   r!  r'  Úinfr   ©r   r   r   )r#  r"  r$  r  Úwelford_reducer5  )r   rÿ   r   r7   Úiinfor!  r   ©r3  r   rF   rF   rG   Údefault_accumulator¾  s*    úùzReduction.default_accumulatorc                 C   s   | dkrdS t  | |¡S )Nr¦  r   )r9  r©  r¨  rF   rF   rG   Údefault_valueØ  s    zReduction.default_value)r£  rV  r<  Úreturnc                 C   sD   | dkr |dkr |t jkr t jS | dkr@|dkr@|t jkr@t jS |S )NrF  i   é   )r  r  r  )r£  rV  r<  rF   rF   rG   Ú_multilayer_second_step_hintÞ  s    ÿþýz&Reduction._multilayer_second_step_hintc                    sD   t  |ˆg¡‰tjj t ˆ| d¡¡ ‰‡ ‡‡‡‡‡fdd„}|S )Nr   c                    sj   |\}| �^ ‰}ˆ| | ‰ ‡ ‡‡‡fdd„}ˆr`t  t  ˆ tj¡t  ˆtj¡¡}t  ||ˆ¡S |ƒ S d S )Nc                      s   ˆˆˆˆ gƒƒS rI   rF   rF   )ru  r  Ú	new_indexr]   rF   rG   Úbody  s    zCReduction._multilayer_wrap_loader.<locals>.wrapper_fn.<locals>.body)r0   r(  rŠ  r7   Zint32Úmasked)rU   rŸ  Zreduction_blockr¯  r0  ©Ú
block_sizeÚdefaultr  Ú	need_maskr~  r]   )ru  r®  rG   Ú
wrapper_fnü  s    
þz5Reduction._multilayer_wrap_loader.<locals>.wrapper_fn)ÚViewÚdynamic_reshape_indexerr1   r�   r‚   rÆ   r;   rÇ   )ré   r  r:  r~  r£  r²  r³  rµ  rF   r±  rG   Ú_multilayer_wrap_loaderí  s    
ÿz!Reduction._multilayer_wrap_loader)	r€   r}  r;  rÒ   rÓ   r:  r3  r£  r<  c
                    sÎ   t |ƒ}
t|
|d  |ƒ}|  ||¡}|  |||
|||¡}|tjtjfkrN|ntj}t 	||||||f•|g||	¡}| 
¡  | ¡ ‰ ‡ fdd„}tjj t |ƒ¡}|  |||	¡}	t 	t|||||g|||	ƒ¡S )úa
        Break a large reduction up into multiple smaller reductions
        recursively
        r   c                    s   ˆ | |•ƒS rI   rF   rž  ©Zintermediate_loaderrF   rG   Úintermediate_fn=  s    z4Reduction.create_multilayer.<locals>.intermediate_fn)r,   r   rª  r¸  r7   Zfloat16Zbfloat16rÿ   r9  rè   rÊ   r÷   r1   r�   r‚   rƒ   r­  r:   )ré   r€   r}  r;  rÒ   rÓ   r:  r3  r£  r<  r~  r²  r³  rµ  Zintermediate_dtypeZintermediater»  rV  rF   rº  rG   r¡    s\         ÿ	ÿýø
  ÿøÿzReduction.create_multilayerN)!r’   r“   r”   r
   r   rË   rP   r7   r   r  rÜ   r‘   rø   rö   r?  rA  rÙ   r  rÌ   r�  r’  rú   r  r€   r   r   rè   r©  rª  rC  r­  r¸  r¡  rF   rF   rF   rG   r9     sh   

		
 B
'
÷
÷y

  þ
!
ör9  c                 C   s   d| krdS dS )NÚwelfordri   r   rF   r=  rF   rF   rG   Únum_reduction_outputsR  s    r½  c                
       s®   e Zd ZU eed< ‡ fdd„Zdd„ Zeej	fe
je
jeedef  ee ee eedœdd	„ƒZed
d„ ƒZee
je
jeedef  ee ee eeedœdd„ƒZ‡  ZS )ÚWelfordReductionÚoutput_indexc	           
   
      sF   t ˆ ƒdkrˆ d }	n‡ fdd„}	tƒ  |||	|||||¡ || _d S )Nr   r   c                    s   t ‡ ‡fdd„ˆD ƒƒS )Nc                 3   s   | ]}|ˆ ˆƒV  qd S rI   rF   ©rS   rO   ©rl   Úreduction_idxrF   rG   rj  i  s     z<WelfordReduction.__init__.<locals>.loader.<locals>.<genexpr>)r6   rÁ  ©Ú	inner_fnsrÁ  rG   r  h  s    z)WelfordReduction.__init__.<locals>.loader)rY   rÞ   r�   r¿  )
r�   r€   r   rÄ  rÓ   r:  r3  r<  r¿  r  rß   rÃ  rG   r�   Y  s    
ø
zWelfordReduction.__init__c              	   C   s:   t  | j| j| j|  ||¡¡}|| j }t  |||ƒ|¡S rI   )r0   r>  r   r;  r3  rÒ   r¿  r?  )r�   r  r  r  r@  ÚvaluesrŸ   rF   rF   rG   r?  w  s    
ü
z WelfordReduction.store_reduction.)r€   r   rÄ  rÓ   r:  r3  r<  c              
      s2  ˆdkst ‚tjj tˆƒ¡}‡‡‡fdd„}	|dkrX|	dƒ}
|	dƒ}|	dƒ}|
||fS |dkr¨‡‡‡fdd„‰ ˆdkr’ˆ ˆd ƒ|	dƒ|	dƒfS t‡ fd	d
„ˆD ƒƒS tjˆˆˆˆd ˆˆˆ|d�\}}ˆt	j
krØ|‰|dkrø|  ˆˆˆˆˆˆ|ˆ¡S ‡‡‡‡‡‡‡fdd„tdƒD ƒ}|D ]}| ¡  �q|S )N>   r5  r¦  c                    s"   ‡ fdd„}t jˆˆ|tˆƒd�S )Nc                    s   t  ˆ t¡S rI   )r0   rþ   r}  ©rl   r”  rF   rG   rÒ   ‘  s    z8WelfordReduction.create.<locals>.const.<locals>.inner_fnr›  )r  rè   r5   )rl   r•  rÒ   )r€   r   rÓ   r”  rG   Úconst�  s    üz&WelfordReduction.create.<locals>.constr   r   c                    s$   ‡ ‡fdd„}t jˆt|tˆƒd�S )Nc                    s   dd„ ˆD ƒ}ˆ | |ƒS )Nc                 S   s   g | ]}t  d ¡‘qS rœ  r�  ru   rF   rF   rG   rW   ¥  s     zKWelfordReduction.create.<locals>.copy.<locals>.inner_fn.<locals>.<listcomp>rF   )rl   rŸ  )r  r:  rF   rG   rÒ   ¤  s    z7WelfordReduction.create.<locals>.copy.<locals>.inner_fnr›  )r  rè   r}  r5   )r  rÒ   )r€   rÓ   r:  ©r  rG   Úcopy£  s    üz%WelfordReduction.create.<locals>.copyr¦  c                 3   s   | ]}ˆ |ƒV  qd S rI   rF   rÀ  )rÉ  rF   rG   rj  ²  s     z*WelfordReduction.create.<locals>.<genexpr>)r3  r~  c                    s(   g | ] }t  tˆ ˆˆˆˆˆˆ|ƒ¡‘qS rF   )r:   rè   r¾  )rS   Z
output_idx)r€   r   rÄ  rÓ   r<  r:  r3  rF   rG   rW   á  s   ôøÿz+WelfordReduction.create.<locals>.<listcomp>ri   )r@   r1   r�   r‚   r   r,   r6   r9  r�  r  r  r¡  rZ   rÊ   )ré   r€   r   rÄ  rÓ   r:  r3  r<  r~  rÇ  ZmeanÚm2Úweightr¢  r£  Úresultsr‹   rF   )rÉ  r€   r   rÄ  rÓ   r<  r:  r3  rG   rè   �  sT    
ø

øózWelfordReduction.createc                 C   s   dS )Nr¥  rF   r¨  rF   rF   rG   rª  ô  s    zWelfordReduction.default_value©r€   r   rÄ  rÓ   r:  r3  r£  r<  c	              
      s,  t ˆƒ‰tjj t ˆˆ d¡¡ }	|	rp|dkrp‡fdd„}
ˆj|ˆ|d t|
dd�t|
dd�f|ˆdˆ|d�S t	ˆˆd  ˆƒ‰ t
 |ˆt‡ ‡‡‡‡fdd	„|D ƒƒ|ˆf•ˆ g||¡}|D ]}| ¡  q¾d
d„ |D ƒ}dd„ ‰tjj t |ƒ¡}ˆ ˆ||¡}t
 |ˆt‡fdd	„|D ƒƒ|ˆgd|¡S )r¹  r   r5  c                    s   t  |ˆ ¡S rI   r˜  )rl   rÂ  rŸ   rá   rF   rG   rþ     s    z4WelfordReduction.create_multilayer.<locals>.constant)rŸ   r   rÍ  c              	   3   s$   | ]}ˆj |ˆˆˆˆ d d�V  qdS )r   )r³  N)r¸  )rS   r  )r²  ré   r~  r:  r£  rF   rG   rj  '  s   	øúz5WelfordReduction.create_multilayer.<locals>.<genexpr>c                 S   s   g | ]}|  ¡ ‘qS rF   )r÷   rR   rF   rF   rG   rW   :  s     z6WelfordReduction.create_multilayer.<locals>.<listcomp>c                 S   s   || |•ƒS rI   rF   )rU   rŸ  r  rF   rF   rG   Úintermediate_loader_fn<  s    zBWelfordReduction.create_multilayer.<locals>.intermediate_loader_fnc                 3   s   | ]}t ˆ | ¡ d �V  qdS )rÈ  N)r   r÷   rR   )rÎ  rF   rG   rj  F  s   ÿ)r,   r1   r�   r‚   rÆ   r;   rÇ   r¡  r   r   r¾  rè   r6   rÊ   rƒ   r­  )ré   r€   r   rÄ  rÓ   r:  r3  r£  r<  r´  rþ   ZintermediatesrT   Z	i_loadersrV  rF   )r²  ré   r   rÎ  r~  r:  r£  rG   r¡  ø  sh    ÿ

ýô	÷ï
  ÿþõz"WelfordReduction.create_multilayer)r’   r“   r”   rC  rË   r�   r?  rú   r  r  r7   r€   r   r   r   r   r
   r   rP   rè   rÌ   rª  r¡  rû   rF   rF   rß   rG   r¾  V  s4   

	øør
÷r¾  c                 C   s.   zt | dd� W dS  tk
r(   Y dS X d S )NF©ÚfreezeT)Úas_storage_and_layoutrÉ   r¢   rF   rF   rG   r…   R  s
    r…   c                 C   s6   zt | dd�\}}| ¡ W S  tk
r0   Y dS X d S ©NFrÏ  )rÑ  Úis_contiguousrÉ   )r‰   Úbufferrd  rF   rF   rG   Ú is_contiguous_storage_and_layoutZ  s
    
rÕ  Fc                 C   s°   t | tƒrt| j|||d�S t | tƒr‚t | jtƒr‚|rv|rV| j ¡  | jj ¡ svt	‚n |dk	rl| j 
|¡ n
| j ¡  | | jjfS t | tƒr¨t| j|d�\}}|| jfS t‚dS )z0Try to simplify x into a StorageBox and a Layout©rÐ  Úwant_contiguousÚstride_orderNrÏ  )r4   r:   rÑ  re  Ú
StorageBoxÚBufferÚfreeze_layoutrd  rÓ  r@   Úfreeze_layout_with_stride_orderrr  ÚReinterpretViewrÉ   )r‰   rÐ  r×  rØ  rÔ  rv   rF   rF   rG   rÑ  b  s.    
ü


þ

rÑ  )r×  c                 C   s8   zt | dd�\}}| |¡W S  tk
r2   Y dS X d S rÒ  )rÑ  Úis_stride_orderedrÉ   )r‰   rØ  rÔ  rd  rF   rF   rG   Ú"is_stride_order_storage_and_layout…  s
    rß  c                   @   s–   e Zd ZU eed< dd„ Zdd„ Zdd„ Zdd	„ Zd
d„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zd d!„ Zd"S )#ÚBaseViewre  c                 C   s   t d| › �ƒ‚d S )Nzmake_reindexer NYI on ©rÉ   rŽ   rF   rF   rG   Úmake_reindexer‘  s    zBaseView.make_reindexerc                    s$   | j  ¡ ‰ |  ¡ ‰‡ ‡fdd„}|S )Nc                    s   ˆ ˆ| ƒƒS rI   rF   rÆ  ©Úinnerr]   rF   rG   r  ˜  s    z&BaseView.make_indexer.<locals>.indexer)re  r‘  râ  ©r�   r  rF   rã  rG   r‘  ”  s    
zBaseView.make_indexerc                    s$   | j  ¡ ‰ |  ¡ ‰‡ ‡fdd„}|S )Nc                    s   ˆ ˆ| ƒƒS rI   rF   rÆ  rã  rF   rG   r  ¡  s    z$BaseView.make_loader.<locals>.loader)re  r÷   râ  ©r�   r  rF   rã  rG   r÷   �  s    
zBaseView.make_loaderc                 C   s
   | j  ¡ S rI   )re  r‡   rŽ   rF   rF   rG   r‡   ¦  s    zBaseView.get_dtypec                 C   s
   | j  ¡ S rI   )re  rˆ   rŽ   rF   rF   rG   rˆ   ©  s    zBaseView.get_devicec                 C   s   d S rI   rF   rŽ   rF   rF   rG   rä   ¬  s    zBaseView.get_origin_nodec                 C   s
   | j  ¡ S rI   ©re  Úget_namerŽ   rF   rF   rG   rè  ¯  s    zBaseView.get_namec                 C   s   | j  |¡S rI   )re  Ú
mark_reuse©r�   ÚusersrF   rF   rG   ré  ²  s    zBaseView.mark_reusec                 C   s
   | j  ¡ S rI   )re  Úhas_exceeded_max_readsrŽ   rF   rF   rG   rì  µ  s    zBaseView.has_exceeded_max_readsc                 C   s
   | j  ¡ S rI   ©re  rÊ   rŽ   rF   rF   rG   rÊ   ¸  s    zBaseView.realizec                 C   s
   | j  ¡ S rI   )re  Úrealize_hintrŽ   rF   rF   rG   rî  »  s    zBaseView.realize_hintc                 C   s
   | j  ¡ S rI   )re  Úget_storage_numelrŽ   rF   rF   rG   rï  ¾  s    zBaseView.get_storage_numelc                 C   s
   | j  ¡ S rI   )re  ræ   rŽ   rF   rF   rG   ræ   Á  s    zBaseView.is_externc              
   C   s:   t  tdd¡�" t|  ¡ |  ¡ ƒjW  5 Q R £ S Q R X d S rò   )r   rô   rõ   r"   r÷   r„   rù   rŽ   rF   rF   rG   rÂ   Ä  s
    þzBaseView.get_readsc                 C   s   | }t |tƒr|j}q|S rI   )r4   rà  re  ©r�   r‰   rF   rF   rG   Úunwrap_viewË  s    
zBaseView.unwrap_viewc                 C   s0   |   ¡ }t td|¡|ƒ}t||  ¡ ||  ¡ ƒS r	  )r÷   r   rô   r  r  r‡   r„   r  rF   rF   rG   r  Ñ  s    zBaseView.constant_to_deviceN)r’   r“   r”   r¨   rË   râ  r‘  r÷   r‡   rˆ   rä   rè  ré  rì  rÊ   rî  rï  ræ   rÂ   rñ  r  rF   rF   rF   rG   rà  �  s"   
		rà  c                   @   sB   e Zd ZU ee ed< edd„ ƒZedd„ ƒZ	dd„ Z
dd	„ Zd
S )r8   r}   c                 C   sˆ   t ttj|ƒƒ}|  ¡ }dgt|ƒt|ƒ  t |ƒ }t|ƒt|ƒksJt‚tt|ƒƒD ],}|| dkrV|| dk	svt‚|| ||< qV|S )zReplace `-1` with correct sizesNéÿÿÿÿ)r5   r¹   r;   r‰  r„   rY   r@   rZ   )r‰   Únew_sizeÚold_sizerT   rF   rF   rG   Ú_normalize_sizeÜ  s    zExpandView._normalize_sizec           
      C   s°   |   ||¡}t|ƒr¦t|ƒ\}}t|ƒt|jƒ }|dks>t‚t d¡g| }t|j	|jƒD ]$\}}| 
|dkrt|nt d¡¡ q\t|j|jt|ƒ||jƒ}	t||	ƒS t||ƒS ©Nr   r   )rõ  r…   rÑ  rY   r}   r@   r;   r   r_   r~   rp  r�  r€   r   r5   ÚoffsetrÝ  r8   )
ré   r‰   ró  ÚstorageÚ
old_layoutÚskipÚ
new_strider~   r}   Ú
new_layoutrF   rF   rG   rè   é  s"    û
zExpandView.createc                 C   s   | j S rI   ©r}   rŽ   rF   rF   rG   r„   ÿ  s    zExpandView.get_sizec                    s4   |   ¡ }| j  ¡ ‰ t|ƒtˆ ƒ ‰‡ ‡fdd„}|S )Nc                    sT   t | ˆd … ƒ} t| ƒtˆ ƒks$t‚ttˆ ƒƒD ]}ˆ | dkr0t d¡| |< q0| S )Nr   r   )r5   rY   r@   rZ   r;   r   )rU   rT   ©Úactualrú  rF   rG   r]     s    z*ExpandView.make_reindexer.<locals>.reindex)r„   re  rY   )r�   Útargetr]   rF   rþ  rG   râ    s
    
	zExpandView.make_reindexerN)r’   r“   r”   r
   r   rË   rÌ   rõ  rú   rè   r„   râ  rF   rF   rF   rG   r8   Ø  s   


r8   c                   @   sB   e Zd ZU ee ed< edd„ ƒZedd„ ƒZdd„ Z	dd	„ Z
d
S )ÚPermuteViewÚdimsc                    s€   |   |¡}t|ƒttt|ƒƒƒks&t‚t|ƒrvt|ƒ\}‰ tˆ jˆ j	‡ fdd„|D ƒ‡ fdd„|D ƒˆ j
ƒ}t||ƒS t||ƒS )Nc                    s   g | ]}ˆ j | ‘qS rF   rý  rR   ©rù  rF   rG   rW   !  s     z&PermuteView.create.<locals>.<listcomp>c                    s   g | ]}ˆ j | ‘qS rF   )r~   rR   r  rF   rG   rW   "  s     )Ú_map_neg_dimsr­   rZ   rY   r@   r…   rÑ  r�  r€   r   r÷  rÝ  r  )ré   r‰   r  rø  rü  rF   r  rG   rè     s    
û
zPermuteView.createc                    s   ‡ fdd„ˆ D ƒS )Nc                    s$   g | ]}|d kr|n
t ˆ ƒ| ‘qS rœ  )rY   )rS   Údim©r  rF   rG   rW   +  s     z-PermuteView._map_neg_dims.<locals>.<listcomp>rF   )ré   r  rF   r  rG   r  )  s    zPermuteView._map_neg_dimsc                    sD   t |  | j¡ƒt tt| jƒƒƒks&t‚| j ¡ ‰ ‡ fdd„| jD ƒS )Nc                    s   g | ]}ˆ | ‘qS rF   rF   rR   rý  rF   rG   rW   0  s     z(PermuteView.get_size.<locals>.<listcomp>)r­   r  r  rZ   rY   r@   re  r„   rŽ   rF   rý  rG   r„   -  s    &
zPermuteView.get_sizec                    s^   dd„ t | jƒD ƒ‰ ‡ fdd„tt| jƒƒD ƒ‰ tˆ ƒttt| jƒƒƒksNt‚‡ fdd„}|S )Nc                 S   s   i | ]\}}||“qS rF   rF   )rS   rT   ÚjrF   rF   rG   rn   3  s      z.PermuteView.make_reindexer.<locals>.<dictcomp>c                    s   g | ]}ˆ | ‘qS rF   rF   rR   ©ÚinvrF   rG   rW   4  s     z.PermuteView.make_reindexer.<locals>.<listcomp>c                    s   ‡ fdd„ˆD ƒS )Nc                    s   g | ]}ˆ | ‘qS rF   rF   rR   r[   rF   rG   rW   8  s     z?PermuteView.make_reindexer.<locals>.reindex.<locals>.<listcomp>rF   r[   r  r[   rG   r]   7  s    z+PermuteView.make_reindexer.<locals>.reindex)rq   r  rZ   rY   r­   r@   )r�   r]   rF   r  rG   râ  2  s
    zPermuteView.make_reindexerN)r’   r“   r”   r
   r   rË   rú   rè   r  r„   râ  rF   rF   rF   rG   r    s   


r  c                   @   sB   e Zd Zeddœdd„ƒZeeejdf dœdd„ƒZ	d	d
„ Z
dS )ÚSqueezeViewN©r  c                   sD  t |ƒrät|ƒ\}}g }g }ˆ d k	rPtˆ tƒs6tdƒ‚dˆ krLˆ t|jƒk sPt‚tt|j|j	ƒƒD ]`\}\}}	ˆ d kr”|dkrÂ| 
|¡ | 
|	¡ qb|ˆ kr²| 
|¡ | 
|	¡ qb|dksbtdƒ‚qbt|j|j|||jƒ}
t||
ƒS ˆ d k�rt |dd„ | ¡ D ƒ¡S | ¡ ˆ  dk�st‚t |‡ fdd„t| ¡ ƒD ƒ¡S d S )Nzexpected integer dim argumentr   r   zexpected squeezed size to be 1c                 S   s   g | ]}|d kr|‘qS ©r   rF   rz   rF   rF   rG   rW   _  s      z&SqueezeView.create.<locals>.<listcomp>c                    s   g | ]\}}|ˆ kr|‘qS rF   rF   ©rS   rT   r{   r  rF   rG   rW   b  s      )r…   rÑ  r4   rC  r@   rY   r}   rq   r_   r~   rp  r�  r€   r   r÷  rÝ  r¶  rè   r„   )ré   r‰   r  rø  rù  ró  rû  rT   r}   r~   rü  rF   r  rG   rè   >  s8    

û

zSqueezeView.create.rý  c                    sV   dd„ | D ƒ}dd„ t | ƒD ƒ‰t| ƒ‰ ttj ttjdf dœ‡ ‡fdd„}||fS )Nc                 S   s   g | ]}|d kr|‘qS r  rF   rz   rF   rF   rG   rW   f  s      z(SqueezeView.squeezer.<locals>.<listcomp>c                 S   s   g | ]\}}|d kr|‘qS r  rF   r  rF   rF   rG   rW   g  s      .)rU   r«  c                    sV   t | ƒt ˆƒks"t| › dˆ› �ƒ‚t d¡gˆ  }tˆ| ƒD ]\}}|||< q<t|ƒS )Nú r   )rY   r@   r;   r   r_   r6   )rU   r®  rl   r{   ©ÚlengthZnot_onerF   rG   r]   j  s
    "
z%SqueezeView.squeezer.<locals>.reindex)rq   rY   r
   r;   r   r   )r}   ró  r]   rF   r  rG   Úsqueezerd  s
    &zSqueezeView.squeezerc                 C   s   t dƒ‚d S )Nzuse SqueezeView.create())r@   )r�   re  rF   rF   rG   r�   s  s    zSqueezeView.__init__)r’   r“   r”   rú   rè   rÌ   r   r;   r   r  r�   rF   rF   rF   rG   r
  =  s
   %r
  c                   @   sZ   e Zd ZU ee ed< edef ed< dd„ Zdd„ Z	dd	„ Z
e
Zed
d„ ƒZdd„ ZdS )ÚGenericViewr}   .r]   c                 C   s   | j S rI   )r]   rŽ   rF   rF   rG   râ  |  s    zGenericView.make_reindexerc                 C   sB   dd„ t t| jƒƒD ƒ}t|  |¡ƒ}dd tt|ƒ¡› d|› �S )Nc                 S   s   g | ]}t d |› �ƒ‘qS )rT   ©r.   )rS   rì   rF   rF   rG   rW   €  s     z+GenericView.reindex_str.<locals>.<listcomp>zlambda ú, z: )rZ   rY   r}   r5   r]   r¸   r¹   rP   )r�   Z	index_oldZ	index_newrF   rF   rG   Úreindex_str  s    zGenericView.reindex_strc                 C   s$   |   | jd| j› �d|  ¡ › �g¡S )Núsize=zreindex=)r¼   re  r}   r  rŽ   rF   rF   rG   rÜ   „  s    ÿzGenericView.__str__c                 C   s   | |t |ƒ|ƒS rI   )r5   )ré   r‰   ró  r]   rF   rF   rG   rè   ‹  s    zGenericView.createc                 C   s   | j S rI   rý  rŽ   rF   rF   rG   r„   �  s    zGenericView.get_sizeN)r’   r“   r”   r
   r   rË   r   r   râ  r  rÜ   r‘   rú   rè   r„   rF   rF   rF   rG   r  w  s   

r  c                   @   sH   e Zd Zedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZed	d
„ ƒZ	dS )r¶  c                 C   s<   t  | ¡} t  |¡}tjjjj}|t  | d¡ƒr8| | } | S rÄ   )r;   r‰  r1   r�   r‚   Ú	shape_envÚevaluate_exprÚLt)rl   r}   r  rF   rF   rG   Úhandle_negative_index•  s    

zView.handle_negative_indexc                    sÀ   t |ttfƒst‚|  | ¡ |¡\‰ }tjj 	ˆ |¡r:|S d|kr^‡ fdd„}| |t|ƒ|ƒS t
|ƒr¤t |jtƒs¤t|ƒ\}}t|j|j|t |¡|jƒ}t||ƒS |  ˆ |¡}| |t|ƒ|ƒS )Nr   c                    s   t dgtˆ ƒ ƒS rÄ   )r6   rY   r[   ©rô  rF   rG   Úfake_reindex©  s    z!View.create.<locals>.fake_reindex)r4   r6   r5   r@   Úresolve_negative_sizer„   r1   r�   r‚   Zstatically_known_list_equalsrÕ  re  ÚExternKernelAllocÚ as_contiguous_storage_and_layoutr�  r€   r   rõ   r�  r÷  rÝ  r·  )ré   r‰   ró  r  rø  rù  rü  r]   rF   r  rG   rè   ž  s,    
 ÿû
zView.createc                 C   sŠ   dd„ |D ƒ}dd„ | D ƒ} t |ƒ}tt|ƒƒD ]8}|| dkr0t d¡||< tt| ƒt|ƒƒ||<  qjq0tjj	 
t| ƒt|ƒ¡ | |fS )Nc                 S   s   g | ]}t jj |¡‘qS rF   ©r1   r�   r‚   r   r‚  rF   rF   rG   rW   À  s     z.View.resolve_negative_size.<locals>.<listcomp>c                 S   s   g | ]}t jj |¡‘qS rF   r   r‚  rF   rF   rG   rW   Á  s     rò  r   )r5   rZ   rY   r;   r   r   r,   r1   r�   r‚   Úguard_equals)rô  ró  rT   rF   rF   rG   r  ¾  s    zView.resolve_negative_sizec              	   C   sZ   z|   ||¡}W nD ttfk
rT   t|ƒg}|   ||¡}|   ||¡}t||ƒ}Y nX |S rI   )Ú_dynamic_reshape_indexerr@   Ú
IndexErrorr,   rh   )ré   rô  ró  r]   Zflatrf   rg   rF   rF   rG   r·  Í  s    
zView.dynamic_reshape_indexerc                    sD  t jjj}dd„ tt|ƒƒD ƒ‰ ttˆ |ƒƒ}t| ƒ}g ‰|�r¸|�r¸| ¡ }| ¡ \}}|dkr‚ˆ 	t
 d¡¡ | 	||f¡ q:|dkr–| 	|¡ q:||ƒ||ƒkrÂˆ 	|¡ t jj ||¡ q:||ƒ||ƒk �r$||ƒ||ƒk �r| ¡ \}}	|| | }||	 }qÔˆ 	|¡ t jj ||¡ q:||ƒ||ƒk�r°t
 d¡}
|}ˆ 	t||
|ƒ¡ |
| }
||ƒ||ƒk�rž| ¡ }ˆ 	t||
|ƒ¡ |
| }
|| }�q^t jj ||¡ q:tƒ ‚q:|�rê| ¡ }t jj |d¡ ˆ 	t
 d¡¡ �q¸|�r| ¡ \}}t jj |d¡ �qêttˆƒƒ‰tˆƒt| ƒk�s2t‚‡ ‡fdd„}|S )zG
        Perform a reshape entirely by modifying indexing math
        c                 S   s   g | ]}t d |› �ƒ‘qS ©Úviewr  rR   rF   rF   rG   rW   ß  s     z1View._dynamic_reshape_indexer.<locals>.<listcomp>r   r   c                    sH   t | ƒt ˆƒks$tt | ƒt ˆƒfƒ‚ttˆ| ƒƒ‰ t‡ fdd„ˆD ƒƒS )Nc                 3   s   | ]}t |ˆ ƒV  qd S rI   ©r-   r‚  ©ÚreplacementsrF   rG   rj    s     zAView._dynamic_reshape_indexer.<locals>.reindex.<locals>.<genexpr>)rY   r@   r^   r_   r6   r[   ©r  Z	view_exprr'  rG   r]     s    $z.View._dynamic_reshape_indexer.<locals>.reindex)r1   r�   r‚   rƒ   rZ   rY   r5   r_   rç   rp  r;   r   r!  r   r@   Úreversed)rô  ró  rƒ   Z	stack_newZ	stack_oldZsize_oldÚvarZsize_newZvar2Z	size_new2ZdivisorÚmodulusr]   rF   r)  rG   r"  Ù  s\    




zView._dynamic_reshape_indexerN)
r’   r“   r”   rÌ   r  rú   rè   r  r·  r"  rF   rF   rF   rG   r¶  “  s   



r¶  c                       sŽ   e Zd ZU dZded< ‡ fdd„Zdd„ ZeZdd	„ Zd
d„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Z‡  ZS )rÝ  z*Pretend our storage has a different layoutÚLayoutrd  c                    s&   t ƒ  ¡  t| jtƒr"| j ¡ | _d S rI   )rÞ   r±   r4   re  rà  rñ  rŽ   rß   rF   rG   r±     s    
zReinterpretView.__post_init__c                 C   s   |   | j| jg¡S rI   )r¼   re  rd  rŽ   rF   rF   rG   rÜ   $  s
    þÿzReinterpretView.__str__c                 C   s
   | j  ¡ S rI   rç  rŽ   rF   rF   rG   rè  .  s    zReinterpretView.get_namec                 C   s   | j jS rI   ©rd  r€   rŽ   rF   rF   rG   rˆ   1  s    zReinterpretView.get_devicec                 C   s   d S rI   rF   rŽ   rF   rF   rG   rä   4  s    zReinterpretView.get_origin_nodec                 C   s   | j jS rI   )rd  r   rŽ   rF   rF   rG   r‡   7  s    zReinterpretView.get_dtypec                 C   s   t | jjƒS rI   ©r5   rd  r}   rŽ   rF   rF   rG   r„   :  s    zReinterpretView.get_sizec                 C   s   t | jjƒS rI   ©r5   rd  r~   rŽ   rF   rF   rG   Ú
get_stride=  s    zReinterpretView.get_stridec                    s   ‡ fdd„}|S )Nc                    s   ˆ j  ¡ }t ˆ  ¡ || ƒ¡S rI   )rd  r‘  r0   Úloadrè  ©rU   r  rŽ   rF   rG   r  A  s    
z+ReinterpretView.make_loader.<locals>.loaderrF   ræ  rF   rŽ   rG   r÷   @  s    zReinterpretView.make_loaderc                 C   s
   | j  ¡ S rI   ©rd  r‘  rŽ   rF   rF   rG   r‘  G  s    zReinterpretView.make_indexerc                 C   s   | j S rI   ©rd  rŽ   rF   rF   rG   r†   J  s    zReinterpretView.get_layoutc                 C   s   d S rI   rF   rŽ   rF   rF   rG   rÛ  M  s    zReinterpretView.freeze_layoutc              	   C   sX   t jj | jj¡}t jj | jj¡}t jj | jj¡}d|  	¡ › d|› d|› d|› d�	S )Nzreinterpret_tensor(r  ú))
r1   r�   Úwrapper_codeÚcodegen_shape_tuplerd  r}   r~   Zcodegen_sizevarr÷  rè  )r�   r}   r~   r÷  rF   rF   rG   Úcodegen_referenceP  s    z!ReinterpretView.codegen_reference)r’   r“   r”   Ú__doc__rË   r±   rÜ   r‘   rè  rˆ   rä   r‡   r„   r1  r÷   r‘  r†   rÛ  r9  rû   rF   rF   rß   rG   rÝ    s    
rÝ  c                   @   s   e Zd Zeddd„ƒZdS )Ú	SliceViewr   c                    s|  t  ˆ¡‰ˆdkst‚z"ˆdkr6|dkr6ˆdkr6|W S W n tk
rL   Y nX tjj}t| ¡ ƒ‰|  	ˆˆˆ  ¡‰|  	|ˆˆ  ¡}| 
|ˆˆ  ¡}| 
| 
ˆˆˆ  ¡|¡‰ˆdkrä| |ˆˆ   ¡dkräˆdkrä| |ˆˆ  ¡ |S t|ˆ ˆd  ˆƒˆˆ < t|ƒ�r\t|ƒ\}}t|jƒ}	|	ˆ  ˆ |	ˆ < t|j|jˆ|	|j|jˆ  ˆ  ƒ}
t||
ƒS ‡ ‡‡‡fdd„}t|ˆ|d�S )Nr   l   ÿÿÿÿ r   c                    sD   t | ƒt ˆƒks$td| › dˆ› �ƒ‚t| ƒ} | ˆ  ˆ ˆ | ˆ < | S )Nzwrong ndim r  )rY   r@   r5   r[   ©r  ró  ÚstartÚsteprF   rG   r]   �  s    $z!SliceView.create.<locals>.reindex)r}   r]   )r;   r‰  r@   Ú	TypeErrorr1   r�   r‚   r5   r„   r  Zevaluate_minrƒ   r!  r   r…   rÑ  r~   r�  r€   r   r÷  rÝ  r;  )ré   r‰   r  r=  Úendr>  r‚   rø  rù  rû  rü  r]   rF   r<  rG   rè   [  s>    

&

û
zSliceView.createN)r   )r’   r“   r”   rú   rè   rF   rF   rF   rG   r;  Z  s   r;  c                   @   sL   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
dS )ÚBaseConstantc                 C   s   dS ©NrF   rF   rŽ   rF   rF   rG   r„   Œ  s    zBaseConstant.get_sizec                 C   s   | j S rI   rá   rŽ   rF   rF   rG   r‡   �  s    zBaseConstant.get_dtypec                 C   s   | j S rI   râ   rŽ   rF   rF   rG   rˆ   ’  s    zBaseConstant.get_devicec                 C   s   d S rI   rF   rŽ   rF   rF   rG   rä   •  s    zBaseConstant.get_origin_nodec                 C   s   d S rI   rF   rê  rF   rF   rG   ré  ˜  s    zBaseConstant.mark_reusec                 C   s   dS rå   rF   rŽ   rF   rF   rG   rì  ›  s    z#BaseConstant.has_exceeded_max_readsc                 C   s   dS rB  rF   rŽ   rF   rF   rG   rÂ   ž  s    zBaseConstant.get_readsc                 C   s   dS rå   rF   rŽ   rF   rF   rG   ræ   ¡  s    zBaseConstant.is_externN)r’   r“   r”   r„   r‡   rˆ   rä   ré  rì  rÂ   ræ   rF   rF   rF   rG   rA  ‹  s   rA  c                   @   sB   e Zd ZU eed< ejed< ejed< dd„ Zdd„ Z	dd	„ Z
d
S )ÚConstantrŸ   r   r€   c                    s   ‡ fdd„}|S )Nc                    s   t  ˆ jˆ j¡S rI   )r0   rþ   rŸ   r   r[   rŽ   rF   rG   r  ¬  s    z$Constant.make_loader.<locals>.loaderrF   ræ  rF   rŽ   rG   r÷   «  s    zConstant.make_loaderc                 C   s   d S rI   rF   rŽ   rF   rF   rG   rÊ   ±  s    zConstant.realizec                 C   s   t | j| j|ƒS rI   )rC  rŸ   r   ©r�   r€   rF   rF   rG   r  ´  s    zConstant.constant_to_deviceN)r’   r“   r”   r   rË   r7   r   r€   r÷   rÊ   r  rF   rF   rF   rG   rC  ¥  s   


rC  c                   @   s:   e Zd ZU eed< ejed< ejed< dd„ Zdd„ Z	dS )	ÚIndexingConstantrU   r   r€   c                    s   ‡ fdd„}|S )Nc                    s   t  ˆ jˆ j¡S rI   )r0   rŠ  rU   r   r[   rŽ   rF   rG   r  ¿  s    z,IndexingConstant.make_loader.<locals>.loaderrF   ræ  rF   rŽ   rG   r÷   ¾  s    zIndexingConstant.make_loaderc                 C   s   t | j| j|ƒS rI   )rE  rU   r   rD  rF   rF   rG   r  Ä  s    z#IndexingConstant.constant_to_deviceN)
r’   r“   r”   r   rË   r7   r   r€   r÷   r  rF   rF   rF   rG   rE  ¸  s
   


rE  c                   @   s¤   e Zd Zedƒfejejee ee edœdd„Z	e
dd„ ƒZdd„ ZeZd	d
„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zedœdd„Zejdœdd„ZdS )r-  r   ©r€   r   r}   r~   r÷  c                 C   sd   |d ks,t |ƒt |ƒks,td|› d|› �ƒ‚|| _|| _tdd„ |D ƒƒsNt‚|| _|| _|| _d S )Nr  ú	, stride=c                 s   s   | ]}t |ttfƒV  qd S rI   )r4   r   rC  rz   rF   rF   rG   rj  ×  s     z"Layout.__init__.<locals>.<genexpr>)rY   r@   r€   r   ro  r}   Ú_strider÷  ©r�   r€   r   r}   r~   r÷  rF   rF   rG   r�   Ê  s    ÿþzLayout.__init__c                 C   s   | j S rI   )rH  rŽ   rF   rF   rG   r~   Ü  s    zLayout.stridec                 C   sP   d}| j dkrd| j › �}t| ƒj› d| jj› d| j› d| j› d| j› |› d�S )	Nr³   r   z	, offset=z('ú', z, size=rG  r6  )r÷  rA   r’   r€   r   r}   r~   )r�   r÷  rF   rF   rG   rÜ   à  s
    
4ÿzLayout.__str__c                 C   s>   t | jt | j¡| jƒD ] \}}}|dkr||kr dS qdS ©Nr   FT)r_   r~   rõ   r�  r}   ©r�   ÚleftÚrightr}   rF   rF   rG   rÓ  ë  s     
 ÿzLayout.is_contiguousc                 C   sR   t | jƒ}|dkrdS t| jt| jƒ| jƒD ] \}}}|dkr,||kr, dS q,dS )N)r^  é   Fr   T)rY   r}   r_   r~   r   )r�   ÚndimrM  rN  r}   rF   rF   rG   Úis_channels_last_contiguousó  s    
  ÿz"Layout.is_channels_last_contiguousc                 C   sB   t | jtt | j¡ƒ| jƒD ] \}}}|dkr||kr dS qdS rK  )r_   r~   r*  rõ   r�  r}   rL  rF   rF   rG   Úis_transposedþ  s    ýzLayout.is_transposedc                 C   s†   t | jƒt |ƒkst‚dgt |ƒ }tt |ƒƒD ] }tjj | j| ¡||| < q0tt |ƒd ƒD ]}|| ||d  krb dS qbdS )Nrò  r   FT)rY   r~   r@   rZ   r1   r�   r‚   rƒ   )r�   ra   Ústride_orderedrT   rF   rF   rG   rÞ    s    zLayout.is_stride_orderedc                 C   s:   dgt ttdt| jƒd ƒƒƒ }t|ƒg| }|  |¡S rö  )r5   r*  rZ   rY   r~   rÞ  ©r�   ra   rF   rF   rG   Úis_channels_last_stride_ordered  s    "z&Layout.is_channels_last_stride_orderedc                 C   s   t | j| j| j| j| jƒS rI   )r�  r€   r   r}   r~   r÷  rŽ   rF   rF   rG   Úas_fixed  s    ûzLayout.as_fixedc                 C   s(   t jstdt| ƒj› d�ƒ‚|  ¡  ¡ S )Nzconvert z to FixedLayout first)rõ   ró   r@   rA   r’   rV  r‘  rŽ   rF   rF   rG   r‘  #  s
    ÿþzLayout.make_indexer©r«  c                 C   s<   | j |j ko:| j|jko:| j|jko:| j|jko:| j|jkS rI   rF  )r�   ÚotherrF   rF   rG   Ú__eq__)  s    
ÿ
þ
ý
ûzLayout.__eq__c                 C   s   t | j| j| jƒS rI   )r   r}   r~   r÷  rŽ   rF   rF   rG   Ústorage_size2  s    zLayout.storage_sizeN)r’   r“   r”   r   r7   r€   r   r
   r   r�   Úpropertyr~   rÜ   r‘   rÓ  rQ  rR  rÞ  rU  rV  r‘  r“  rY  r;   rZ  rF   rF   rF   rG   r-  È  s*   úú
	
		r-  c                	       sp   e Zd ZdZdedƒfejejee	e
 e	e f eee	e
 e	e f  ee
ef dœ‡ fdd„Zdd„ Z‡  ZS )	r�  z A Tensor layout we cannot changeNr   rF  c                    s*   |d krt  |¡}tƒ  |||||¡ d S rI   )rõ   r�  rÞ   r�   rI  rß   rF   rG   r�   9  s    
ûzFixedLayout.__init__c                    s   ‡ fdd„}|S )z1A closure containing math to read a given elementc                    sd   t | ƒt ˆ jƒ  kr$t ˆ jƒks*n t‚ˆ j}t| ˆ jˆ jƒD ]\}}}|dkr@|||  }q@|S rŽ  )rY   r~   r}   r@   r÷  r_   )rU   Úresultrl   r~   ÚszrŽ   rF   rG   r  N  s    *z)FixedLayout.make_indexer.<locals>.indexerrF   rå  rF   rŽ   rG   r‘  K  s    zFixedLayout.make_indexer)r’   r“   r”   r:  r   r7   r€   r   r   r
   r   rC  r   r�   r‘  rû   rF   rF   rß   rG   r�  6  s   ú
úr�  c                       sn   e Zd ZdZdZedd„ ƒZedd„ ƒZedd„ ƒZed	d
„ ƒZ	dd„ Z
dd„ Zdd„ Zd‡ fdd„	Z‡  ZS )rõ   z(A Tensor layout we are allowed to changeFc                 C   sP   t | ƒdkrg S t d¡g}t| dd … ƒD ]}| ||d  ¡ q,tt|ƒƒS )Nr   r   rò  )rY   r;   r   r*  rp  r5   )ÚsizesZreversed_stridesr}   rF   rF   rG   r�  ^  s    z!FlexibleLayout.contiguous_stridesc                 C   sV   t tt| ƒƒƒt |ƒkst‚t d¡}dgt|ƒ }|D ]}|||< || |  }q8|S )z¢
        Create a stride based on the order the dimensions should be filled in.

        In this format, channels last would be:
            [1, 3, 2, 0]
        r   N)r­   rZ   rY   r@   r;   r   )r^  ra   Znext_strider  rT   rF   rF   rG   Úfill_orderedg  s    
zFlexibleLayout.fill_orderedc                 C   s0   t tt| ƒƒƒt |ƒkst‚t|ƒ}t | |¡S )zš
        Create a stride based on the sorted order of a permuted range.

        In this format, channels last would be:
            [3, 0, 2, 1]
        )r­   rZ   rY   r@   rs   rõ   r_  )r^  ra   rr   rF   rF   rG   rS  x  s    zFlexibleLayout.stride_orderedc                 C   sD   t | ƒt |ƒkst‚dd„ |D ƒ}ttt |ƒƒ|jd�}t | |¡S )z»
        Create a stride that has the same stride order as given stride

        For example, if given stride is [1000, 1, 100, 10],
        the fill order should be [1, 3, 2, 0]
        c                 S   s   g | ]}t jj |¡‘qS rF   )r1   r�   r‚   rƒ   r‚  rF   rF   rG   rW   �  s     z/FlexibleLayout.same_ordered.<locals>.<listcomp>rM  )rY   r@   rn  rZ   Ú__getitem__rõ   r_  )r^  r~   rr   rF   rF   rG   Úsame_ordered„  s    zFlexibleLayout.same_orderedc                 C   s"   t | j| j| j|  | j|¡| jƒS rI   )r�  r€   r   r}   rS  r÷  rT  rF   rF   rG   Úas_stride_order‘  s    ûzFlexibleLayout.as_stride_orderc                 C   s"   t | j| j| j|  | j|¡| jƒS rI   )r�  r€   r   r}   r_  r÷  rT  rF   rF   rG   Úas_fill_orderš  s    ûzFlexibleLayout.as_fill_orderc                 C   s"   t | j| j| j|  | j|¡| jƒS rI   )r�  r€   r   r}   ra  r÷  ©r�   r~   rF   rF   rG   Úas_same_order£  s    ûzFlexibleLayout.as_same_orderNc                    s2   |rt  ||¡}n
t  |¡}tƒ  ||||¡ d S rI   )rõ   r_  r�  rÞ   r�   )r�   r€   r   r}   rØ  r  rß   rF   rG   r�   ¬  s    
zFlexibleLayout.__init__)N)r’   r“   r”   r:  ró   rÌ   r�  r_  rS  ra  rb  rc  re  r�   rû   rF   rF   rß   rG   rõ   Y  s   



			rõ   c                       s6   e Zd ZdZddœ‡ fdd„Zdd„ Zdd	„ Z‡  ZS )
ÚAliasedLayoutz)Shares the same storage as another tensorrÝ  r$  c                    s,   |  ¡ }tƒ  |j|j|j|j¡ || _d S rI   )r†   rÞ   r�   r€   r   r}   r~   r%  )r�   r%  rd  rß   rF   rG   r�   ·  s    üzAliasedLayout.__init__c                 C   s   |   ¡  ¡ S rI   )rV  r‘  rŽ   rF   rF   rG   r‘  Á  s    zAliasedLayout.make_indexerc                 C   s4   | j  ¡ j}|dkrdS ddlm} tjj ||¡S )Nr   Tr   )Ú	ALIGNMENT)	r%  r†   r÷  Z
compile_fxrg  r1   r�   r‚   Zstatically_known_multiple_of)r�   r÷  rg  rF   rF   rG   Úmaybe_guard_alignedÄ  s
    z!AliasedLayout.maybe_guard_aligned)r’   r“   r”   r:  r�   r‘  rh  rû   rF   rF   rß   rG   rf  ´  s   
rf  c                       st   e Zd Zedœ‡ fdd„Zejjdd„ ƒZej	dœdd„Z
d	dœd
d„Zdd„ Zedd„ ƒZdd„ Zdd„ Z‡  ZS )ÚMutationLayout©r   c                    s@   t ƒ  | ¡ | ¡ | ¡ d ¡ || _|  ¡  ¡ }tj	 
|¡ d S rI   )rÞ   r�   rˆ   r‡   r„   r   Ú
get_bufferrè  r1   r�   Zmark_buffer_mutated)r�   r   rN   rß   rF   rG   r�   Î  s    üzMutationLayout.__init__c                 C   s
   |   ¡ jS rI   )Úreal_layoutr~   rŽ   rF   rF   rG   r~   Ù  s    zMutationLayout.striderW  c                 C   s   |   ¡  ¡ S rI   )rl  rZ  rŽ   rF   rF   rG   rZ  Ý  s    zMutationLayout.storage_sizerÚ  c                    s,   ‡ fdd„‰ ˆ | j ƒ}t|tƒs(tdƒ‚|S )Nc                    sB   t | tƒrˆ | jƒS t | tƒr*ˆ |  ¡ ƒS t | tƒr>ˆ | jƒS | S rI   )r4   ri  r   rà  rñ  Ú
MutableBoxre  rj  ©Úunwrap_viewsrF   rG   ro  á  s    




z/MutationLayout.get_buffer.<locals>.unwrap_viewsz%MutationLayout must refer to a buffer)r   r4   rÚ  r@   )r�   r\  rF   rn  rG   rk  à  s    	
zMutationLayout.get_bufferc                 C   s
   |   ¡ jS rI   )rk  rd  rŽ   rF   rF   rG   rl  î  s    zMutationLayout.real_layoutc              	   C   s¶   |  ¡  t|tƒr|j}t|tƒr0| | ¡ ¡r6d}n|  ¡  t|jjtƒ }|r’t	j
| ¡ | ¡ | ¡ dd„ t| ¡ | ¡ ƒD ƒd�j}|  ¡  t|jjtƒs¤t‚t|ƒ|j_|jS )NTc                 S   s    g | ]\}}t jj ||¡‘qS rF   ©r1   r�   r‚   r!  ©rS   r.  r/  rF   rF   rG   rW   	  s   ÿz/MutationLayout.realize_into.<locals>.<listcomp>r›  )rÊ   r4   r:   re  rÙ  r¿   rè  rd  rõ   r  rè   rˆ   r‡   r÷   r_   r„   r@   ri  )ré   ÚsrcÚdstZ	need_copyrF   rF   rG   Úrealize_intoñ  s(    
þü	zMutationLayout.realize_intoc                 C   s   | S rI   rF   rŽ   rF   rF   rG   rV  	  s    zMutationLayout.as_fixedc                 C   s
   | j  ¡ S rI   )r   r‘  rŽ   rF   rF   rG   r‘  	  s    zMutationLayout.make_indexer)r’   r“   r”   r¨   r�   r-  r~   Úgetterr;   r   rZ  rk  rl  rú   rt  rV  r‘  rû   rF   rF   rß   rG   ri  Í  s   

ri  c                       sò   e Zd ZU ee ed< eed< ‡ fdd„Zdd„ Zdd„ Z	d	d
„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd „ Zd!d"„ Zd#d$„ Zd%d&„ Zd'd(„ Zd)d*„ Zd+d,„ Zd-d.„ Zd/d0„ Zd1d2„ Zd3d4„ Z‡  Z S )5rÚ  rN   rd  c                    s   t ƒ  ¡  d | _d S rI   rÝ   rŽ   rß   rF   rG   r±   	  s    
zBuffer.__post_init__c                 C   s
   | j  ¡ S rI   r4  rŽ   rF   rF   rG   r‘   	  s    zBuffer.make_indexerc                 C   s   | j s
t‚| j S rI   )rN   r@   rŽ   rF   rF   rG   rè  #	  s    
zBuffer.get_namec                 C   s   | j jS rI   r.  rŽ   rF   rF   rG   rˆ   '	  s    zBuffer.get_devicec                 C   s   | j S rI   rã   rŽ   rF   rF   rG   rä   *	  s    zBuffer.get_origin_nodec                 C   s   t | jdd ƒS )Nr   )rJ   rd  rŽ   rF   rF   rG   r‡   -	  s    zBuffer.get_dtypec                 C   s   t | jjƒS rI   r/  rŽ   rF   rF   rG   r„   0	  s    zBuffer.get_sizec                 C   s   t | jjƒS rI   r0  rŽ   rF   rF   rG   r1  3	  s    zBuffer.get_stridec                 C   s   | j S rI   r5  rŽ   rF   rF   rG   r†   6	  s    zBuffer.get_layoutc                 C   s   |   ¡ S rI   )rÃ   rŽ   rF   rF   rG   rï  9	  s    zBuffer.get_storage_numelc                 C   s   dS rå   rF   rŽ   rF   rF   rG   ræ   <	  s    zBuffer.is_externc                 C   s    t | jttfƒs| j ¡ | _d S rI   )r4   rd  ÚMultiOutputLayoutrf  rV  rŽ   rF   rF   rG   rÛ  ?	  s    zBuffer.freeze_layoutc                 C   s"   t | jtƒst‚| j |¡| _d S rI   )r4   rd  rõ   r@   rb  rT  rF   rF   rG   rÜ  C	  s    z&Buffer.freeze_layout_with_stride_orderc                 C   s"   t | jtƒst‚| j |¡| _d S rI   )r4   rd  rõ   r@   rc  rT  rF   rF   rG   Úfreeze_layout_with_fill_orderG	  s    z$Buffer.freeze_layout_with_fill_orderc                 C   s"   t | jtƒst‚| j |¡| _d S rI   )r4   rd  rõ   r@   re  rd  rF   rF   rG   Úfreeze_layout_with_same_orderK	  s    z$Buffer.freeze_layout_with_same_orderc                 C   s   t jj t |  ¡ d¡¡S rÄ   rÅ   rŽ   rF   rF   rG   rÈ   O	  s    zBuffer.is_zero_elementsc                    s(   ˆ   ¡ rttˆ  ¡ d�S ‡ fdd„}|S )Nrá   c                    s   ˆ j  ¡ }t ˆ j|| ƒ¡S rI   )rd  r‘  r0   r2  rN   r3  rŽ   rF   rG   r  W	  s    
z"Buffer.make_loader.<locals>.loader)rÈ   r   r   r‡   ræ  rF   rŽ   rG   r÷   R	  s    zBuffer.make_loaderc                 C   s   dS rå   rF   rŽ   rF   rF   rG   Úis_no_op]	  s    zBuffer.is_no_opc                 C   s   |   ¡ S rI   )rè  rŽ   rF   rF   rG   r9  `	  s    zBuffer.codegen_referencec                 C   s   d S rI   rF   rŽ   rF   rF   rG   rr  c	  s    zBuffer.decide_layoutc                 C   s   t | jtƒr| jj ¡ gS dS rB  )r4   rd  rf  r%  rè  rŽ   rF   rF   rG   Úget_alias_namesf	  s    zBuffer.get_alias_namesc                 C   s   t | jtƒr| jj ¡ gS dS rB  )r4   rd  ri  r   rè  rŽ   rF   rF   rG   Úget_mutation_namesk	  s    zBuffer.get_mutation_namesc              
   C   s8   t  tdd¡�  t|  ¡ |  ¡ ƒW  5 Q R £ S Q R X d S rò   )r   rô   rõ   r"   r÷   r„   rŽ   rF   rF   rG   rl  p	  s
    þzBuffer.get_read_writesc                 C   s
   |   ¡ jS rI   )rl  rù   rŽ   rF   rF   rG   rÂ   w	  s    zBuffer.get_readsc                 C   s   d S rI   rF   rŽ   rF   rF   rG   rÊ   z	  s    zBuffer.realize)!r’   r“   r”   r   rP   rË   r-  r±   r‘  rè  rˆ   rä   r‡   r„   r1  r†   rï  ræ   rÛ  rÜ  rw  rx  rÈ   r÷   ry  r9  rr  rz  r{  rl  rÂ   rÊ   rû   rF   rF   rß   rG   rÚ  	  s6   
rÚ  c                   @   s   e Zd ZdS )ÚInputBufferN)r’   r“   r”   rF   rF   rF   rG   r|  ~	  s   r|  c                   @   s    e Zd ZdZdd„ Zdd„ ZdS )r  Nc                    s   ‡ fdd„}|S )Nc                    s(   ˆ j  ¡ }t tj ˆ jˆ j¡|| ƒ¡S rI   )	rd  r‘  r0   r2  r1   r�   Úconstant_namerN   r  r3  rŽ   rF   rG   r  †	  s
    
 ÿz*ConstantBuffer.make_loader.<locals>.loaderrF   ræ  rF   rŽ   rG   r÷   …	  s    zConstantBuffer.make_loaderc                 C   s   t tj | j|¡| jƒS rI   )r  r1   r�   r}  rN   rd  rD  rF   rF   rG   r  Ž	  s    z!ConstantBuffer.constant_to_device)r’   r“   r”   r  r÷   r  rF   rF   rF   rG   r  ‚	  s   	r  c                   @   s   e Zd Zdd„ ZdS )ÚNoneAsConstantBufferc                 C   s
   t jjjS rI   )r1   r�   r7  Únone_strrŽ   rF   rF   rG   r9  “	  s    z&NoneAsConstantBuffer.codegen_referenceN)r’   r“   r”   r9  rF   rF   rF   rG   r~  ’	  s   r~  c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚShapeAsConstantBufferc                    s   t ƒ  ¡  || _d S rI   )rÞ   r�   Úshape)r�   r�  rß   rF   rG   r�   ˜	  s    
zShapeAsConstantBuffer.__init__c                 C   s6   t jj t jj | j¡¡}t jjr.d|› d�S |S d S )Nztorch::tensor(r6  )r1   r�   r7  Zexpr_printerr‚   r   r�  rž   )r�   ÚexprrF   rF   rG   r9  œ	  s    z'ShapeAsConstantBuffer.codegen_reference)r’   r“   r”   r�   r9  rû   rF   rF   rß   rG   r€  —	  s   r€  c                       s�   e Zd ZU eed< edd„ ƒZdd„ Z‡ fdd„Zdd	„ Z	d
d„ Z
dd„ Zdd„ Zeddd„ƒZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Z‡  ZS )rk  re  c                 C   s   t |  ¡ jƒS rI   )rY   rl  rù   rŽ   rF   rF   rG   Ú	num_reads©	  s    zComputedBuffer.num_readsc              
   C   sp   t  tdd¡�X | j ¡ rBt|  ¡ | j ¡ | j ¡ ƒW  5 Q R £ S t|  ¡ | j ¡ ƒW  5 Q R £ S W 5 Q R X d S rò   )	r   rô   rõ   re  rö   r"   Úget_store_functionr„   rø   rŽ   rF   rF   rG   rl  ­	  s    
ýþzComputedBuffer.get_read_writesc                    s>   t | jdƒo$| jtjjko$|  ¡ dk}|r4| j ¡ S tƒ  ¡ S )Nr÷   r   )	Úhasattrre  rN   r1   r�   Zmutated_buffersrƒ  r÷   rÞ   )r�   Z
can_inlinerß   rF   rG   r÷   »	  s    ÿ
ý
zComputedBuffer.make_loaderc                 C   s@   | j  ¡  ¡ }| j ¡ r*t| jj| j|ƒS t| jj| j|ƒS d S rI   )	rd  rV  r‘  re  rö   r   r?  rN   r  rå  rF   rF   rG   r„  Æ	  s    
z!ComputedBuffer.get_store_functionc                    s    t | jtƒrœt | j ¡ | j ¡ ¡\\‰ ‰}|  ¡ j	}dd„ |D ƒ}t
dd„ |D ƒƒsZt‚‡fdd„|D ƒ}|rœ‡ fdd„|D ƒ}ddlm} |||  ¡ ƒS d	S )
al  
        If our layout is still flexible, try to determine the stride order based on stride orders of reads.

        TODO(jansel): A better algorithm here would look at downstream consumers of this
                      value and try to do global graph-level layout optimization.
                      This is also something just begging to be autotuned.
        c                 S   s0   g | ](}|j tjj ¡ kr(tjj|j  nd ‘qS rI   )rN   r1   r�   rq  r|  rg  rF   rF   rG   rW   Ú	  s   þÿz1ComputedBuffer.get_fill_order.<locals>.<listcomp>c                 s   s    | ]}t |tjtjfƒV  qd S rI   )r4   r   ÚStarDepÚ	MemoryDeprg  rF   rF   rG   rj  â	  s   ÿz0ComputedBuffer.get_fill_order.<locals>.<genexpr>c                    s.   g | ]&}t |tjƒrt|jd d„ ˆ D ƒƒ‘qS )c                 S   s    i | ]}|d kr|t  d ¡“qS rœ  r�  ©rS   ÚvrF   rF   rG   rn   è	  s       z<ComputedBuffer.get_fill_order.<locals>.<listcomp>.<dictcomp>)r4   r   r‡  r-   rU   rg  )r@  rF   rG   rW   æ	  s   ü ÿc                    s   g | ]}t jj |ˆ ¡‘qS rF   ©r1   r�   r‚   r{  ©rS   r‚  ©Ú
index_varsrF   rG   rW   ï	  s    r   ©Úpick_loop_orderN)r4   rd  rõ   r   ry  re  r„   rø   rl  rù   ro  r@   Ú	schedulerr�  )r�   rv   rù   Ú
reads_bufsZstride_lengthsr�  rF   )r�  r@  rG   Úget_fill_orderÍ	  s,     ÿ
üþ
ü
ÿzComputedBuffer.get_fill_orderc                 C   s0   t | jtƒr,|  ¡ }|r$|  |¡ n|  ¡  d S rI   )r4   rd  rõ   r’  rw  rÛ  rT  rF   rF   rG   rr  ø	  s
    zComputedBuffer.decide_layoutc              	      sê  t jˆj ¡ ˆj ¡ dd�\}}t tdˆ ¡ ¡�* t	ˆ 
¡ ˆ ¡ rH|n
|dd… |ƒ}W 5 Q R X |j ¡ •‰ dd„ |j ¡ D ƒ}|j ¡ |j ¡ •‰g }g }g }g }| ¡ D ]V\}	}
|	|d krà|rÊt‚| |	¡ | |
¡ q®|	|d ksðt‚| |	¡ | |
¡ q®ttt|ƒƒƒgtˆƒ }t|ƒD ],\}}t|tƒ�r(t|d	ƒ�r(|j||< �q(d‡ ‡‡fd
d„	}|| }|||||ƒ\}}}||||ƒ\}}}t|ƒt|ƒk�r®|ˆ_t j||dd�\\}}}t	|||ƒ||ƒg|ƒ}||f|fS )a  
        This is a main place where we do loop transformations in a
        backend-agnostic way.

        Here we:
            1) Remove any 1 dimensions
            2) Fuse contiguous dimensions together
            3) Reorder dimensions based on stride orders
        Úqr2   r  Nr   c                 S   s,   g | ]$}|t jj ¡ kr$t jj| nd ‘qS rI   )r1   r�   rq  r|  )rS   Z
reads_namerF   rF   rG   rW   
  s   þÿz7ComputedBuffer.simplify_and_reorder.<locals>.<listcomp>r   Úiter_reordering_reindexc           	         s\   ˆ  | ||ˆ|¡\}}}|| ƒ} tjj | |tˆ | |ƒ¡\}}}|| ƒ} t||ƒ}|||fS rI   )Ú_apply_loop_reorderingr1   r�   r‚   Ú_simplify_loopsr    rh   )	Zx_varsÚsupport_varsr^  Úreordering_reindexZreindex0rf   rg   Úpruner]   ©Zindex_formulasÚmemory_addrsr�   rF   rG   Úsimplify_and_reorder4
  s         ÿ

ý

zAComputedBuffer.simplify_and_reorder.<locals>.simplify_and_reorderÚz)N)r   ry  re  r„   rø   r   rô   r  rˆ   ÚLoopBodyr„  rö   Úindexing_exprsrÅ  Úreads_name2exprr|  Úwrites_name2exprÚitemsr@   rp  rd   rZ   rY   rq   r4   rk  r…  r”  Zindex_vars_no_squeeze)r�   rK   Ú
var_rangesr¯  r‘  r�  Zreduce_varsZ
index_sizeZreduce_sizer‰  r{   r˜  rT   Z	reads_bufrœ  r—  Ziter_rangesZiter_reindexr”  Zreduce_rangesZreduce_reindexrv   Z	iter_varsrF   rš  rG   rœ   
  s~    
  ÿ
ýüþ

 ÿ   ÿ
  ÿ
  ÿ  ÿz#ComputedBuffer.simplify_and_reorderNc           
   
      s  ddl m} |dkrg }zœ‡ ‡fdd„|D ƒ}t|ƒt|ƒkrRt|d ƒtˆ ƒksVt‚|dk	ržtt|ƒƒD ]2}z|| || ƒ||< W qj tk
rš   Y qjX qjtt||ˆ|ƒƒƒ}	W nB tk
rö   tj	rât
 dttˆ ˆƒƒ|¡ tttˆƒƒƒ}	Y nX ‡fdd„|	D ƒ‰ˆt|	ƒt|	ƒfS )	zU
        Shuffle the order of loops around to hopefully improve performance.
        r   rŽ  Nc                    s   g | ]}t jj |ˆ ˆ¡‘qS rF   rŠ  r‹  )r�  r—  rF   rG   rW   l
  s   ÿz9ComputedBuffer._apply_loop_reordering.<locals>.<listcomp>r   z%Did not simplify complex index:
%s
%sc                    s   g | ]}ˆ | ‘qS rF   rF   rR   )r^  rF   rG   rW   „
  s     )r�  r�  rY   r@   rZ   r5   r*  Ú	Exceptionr   ÚdebugÚlogÚwarningr^   r_   rd   rb   )
r�  r—  r^  r›  r˜  Zpriority_idxr�  r  rT   ra   rF   )r�  r^  r—  rG   r•  Z
  s6    þÿ
ýz%ComputedBuffer._apply_loop_reorderingc                 C   s
   | j  ¡ S rI   )re  rø   rŽ   rF   rF   rG   rø   ‡
  s    z!ComputedBuffer.get_reduction_sizec                 C   s
   | j  ¡ S rI   )re  rö   rŽ   rF   rF   rG   rö   Š
  s    z!ComputedBuffer.get_reduction_typec                 C   s
   | j  ¡ S rI   )re  rÈ   rŽ   rF   rF   rG   ry  �
  s    zComputedBuffer.is_no_opc                 C   s   dS ©NTrF   rŽ   rF   rF   rG   Úshould_allocate�
  s    zComputedBuffer.should_allocatec                 C   s   | j  |¡S )r
  )re  r  rD  rF   rF   rG   r  “
  s    z!ComputedBuffer.constant_to_device)NN)r’   r“   r”   rÑ   rË   r%   rƒ  rl  r÷   r„  r’  rr  rœ  rÌ   r•  rø   rö   ry  r©  r  rû   rF   rF   rß   rG   rk  ¥	  s$   

+Z  ú,rk  c                       sX   e Zd ZdZ‡ fdd„Zdd„ Zdd„ Zdd	„ Zd
d„ Zdd„ Z	dd„ Z
dd„ Z‡  ZS )ÚTemplateBufferzt
    Represents a Triton (in the future other type) of template operator
    that we can fuse an epilogue onto.
    c                    s4   t ƒ jd |d� t |¡| _|| _tj | ¡| _	d S )N)rN   rd  )
rÞ   r�   ÚInputsKernelÚunwrap_storageÚinputsÚmake_kernel_renderr1   r�   Úregister_bufferrN   )r�   rd  r­  r®  rß   rF   rG   r�   ž
  s    zTemplateBuffer.__init__c                 C   s   |   ¡ S rI   )Únormalized_read_writesrŽ   rF   rF   rG   rl  ¤
  s    zTemplateBuffer.get_read_writesc                    sL   |   ¡ ‰| j ¡ ‰ ‡ ‡fdd„}tj||  ¡ ddd�}dd„ | jD ƒ|_|S )Nc                    s"   t |ƒdkst‚t ˆˆ | ƒd¡S )Nr   Zfake)rY   r@   r0   r  rŒ  ©r  rN   rF   rG   Údummy«
  s    z4TemplateBuffer.normalized_read_writes.<locals>.dummyrF   T)Ú	normalizec                 S   s   h | ]}t  | ¡ ¡’qS rF   ©r   r†  rè  r‚  rF   rF   rG   rÁ   ²
  s     z8TemplateBuffer.normalized_read_writes.<locals>.<setcomp>)rè  rd  r‘  r   r"   r„   r­  rù   )r�   r²  ÚdepsrF   r±  rG   r°  §
  s    
   ÿz%TemplateBuffer.normalized_read_writesc                 C   s   dS rŽ  rF   rŽ   rF   rF   rG   rø   µ
  s    z!TemplateBuffer.get_reduction_sizec                 C   s   d S rI   rF   rŽ   rF   rF   rG   rö   ¸
  s    z!TemplateBuffer.get_reduction_typec                 C   s   dS rå   rF   rŽ   rF   rF   rG   ry  »
  s    zTemplateBuffer.is_no_opc                 C   s   dS r¨  rF   rŽ   rF   rF   rG   r©  ¾
  s    zTemplateBuffer.should_allocatec                 C   s   |   ¡ dfd fS rB  )r„   rŽ   rF   rF   rG   rœ  Á
  s
    þûz#TemplateBuffer.simplify_and_reorder)r’   r“   r”   r:  r�   rl  r°  rø   rö   ry  r©  rœ  rû   rF   rF   rß   rG   rª  ˜
  s   rª  c                   @   sJ   e Zd ZU ee ed< dd„ Zdd„ Zedd„ ƒZ	edd	„ ƒZ
d
d„ ZdS )r«  r­  c                 C   s   t  | ¡ ¡S rI   r´  rð  rF   rF   rG   Úget_read_writes_inputÏ
  s    z"InputsKernel.get_read_writes_inputc                    sp   g }ˆ j D ]8}t|tƒr2| ‡ fdd„|D ƒ¡ q
| ˆ  |¡¡ q
tjt|ƒt 	ˆ  
¡ ¡htƒ g d t ¡ d�S )Nc                    s   g | ]}ˆ   |¡‘qS rF   )r¶  r‚  rŽ   rF   rG   rW   Ö
  s     z0InputsKernel.get_read_writes.<locals>.<listcomp>)Z	op_counts)r­  r4   r5   Úextendrp  r¶  r   Z
ReadWritesr­   r†  rè  ÚcollectionsÚCounter)r�   Zstar_depÚinputrF   rŽ   rG   rl  Ò
  s    

úzInputsKernel.get_read_writesc                 C   sX   t | tƒr| j} t | tƒr | j} t | tƒr>t | tƒs>t | ¡} t | ttfƒsTt	| ƒ‚| S rI   )
r4   r:   re  rÙ  rà  rÝ  ÚExternKernelÚrealize_inputrÚ  r@   r¢   rF   rF   rG   Úunwrap_storage_for_inputã
  s    


z%InputsKernel.unwrap_storage_for_inputc                 C   s@   g }| D ]2}t |tƒr&dd„ |D ƒ}n
t |¡}| |¡ q|S )Nc                 S   s   g | ]}t  |¡‘qS rF   )r«  r½  rR   rF   rF   rG   rW   ó
  s     z/InputsKernel.unwrap_storage.<locals>.<listcomp>)r4   r5   r«  r½  rp  )r­  Z
inputs_newr‰   rF   rF   rG   r¬  î
  s    

zInputsKernel.unwrap_storagec                 C   s   dS r¨  rF   rŽ   rF   rF   rG   ræ   ù
  s    zInputsKernel.is_externN)r’   r“   r”   r
   rÚ  rË   r¶  rl  rÌ   r½  r¬  ræ   rF   rF   rF   rG   r«  Ë
  s   




r«  c                   @   s   e Zd Zdd„ ZdS )Ú	NopKernelc                 C   s   dS r¨  rF   rŽ   rF   rF   rG   ry  þ
  s    zNopKernel.is_no_opN)r’   r“   r”   ry  rF   rF   rF   rG   r¾  ý
  s   r¾  c                   @   s0   e Zd ZdZedd„ ƒZedd„ ƒZdd„ ZdS )	ÚConcatKernelzn
    There isn't actually a real kernel for concat, we just change the
    storage for the upstream data.
    c                 C   s  |d   ¡ }|d  ¡ }t|d  ¡ ƒ}dg}|| g}d|  krPt|ƒk sVn t‚tdt|ƒƒD ]´}||  ¡ }	| || ¡ t|	ƒt|ƒks–t‚||  ¡ |ksªt‚||   ¡ |ks¾t‚tt|ƒƒD ]>}
|
|krì||
 |	|
  ||
< qÊtj	j
 ||
 |	|
 ¡||
< qÊ| || ¡ qdt |¡}tt|ƒƒD ]D}|| }t|ƒ�r0| ¡ }t|tƒ�r0| ¡ �r0t|ƒ} �qv�q0td t||||d�g d�}t|ƒ}tt|ƒƒD ]4}|jj |  || t |||| || ¡¡¡ �q¤tj	 |j¡|j_|  |jj¡|j_|S )Nr   r   )r€   r   r}   r~   ©rN   rd  r­  )rˆ   r‡   r5   r„   rY   r@   rZ   rp  r1   r�   r‚   r!  rõ   r�  r…   r†   r4   r�  rQ  r   r¿  rÙ  re  r­  rt  r;  rè   r¯  rN   r¬  )ré   r­  r  r€   r   ró  Zoffsets_startZoffsets_endrT   Ú
input_sizer  Úoutput_strider‰   rd  Zconcat_kernelÚkernelrF   rF   rG   rè     sh    
 ÿ


ÿþ
üø
þÿzConcatKernel.createc              	   C   sÒ   t |tƒs(t|ƒr(t|ƒ\}}t||ƒ}t |tƒs:t|ƒ‚t |tƒrR|  |j|¡S t |tƒr�| 	¡  t |jj
tƒr�t |jtƒs�t|ƒ|j_
|jS tj| ¡ | ¡ | ¡ dd„ t| ¡ | ¡ ƒD ƒd�}|  ||¡S )Nc                 S   s    g | ]\}}t jj ||¡‘qS rF   rp  rq  rF   rF   rG   rW   ^  s   ÿz-ConcatKernel.realize_into.<locals>.<listcomp>r›  )r4   rÝ  r…   rÑ  r@   r:   rt  re  rÙ  rÊ   rd  rõ   r  rf  r  rè   rˆ   r‡   r÷   r_   r„   )ré   rr  rs  rø  rd  ÚpwrF   rF   rG   rt  D  s0    



 ÿþü	zConcatKernel.realize_intoc                 C   s   dS r¨  rF   rŽ   rF   rF   rG   r©  e  s    zConcatKernel.should_allocateN)r’   r“   r”   r:  rú   rè   rt  r©  rF   rF   rF   rG   r¿    s   
;
 r¿  c                   @   s  e Zd ZU dZeedf ed< eje	d�Z
eeef ed< dZee ed< dd	„ Zd
d„ Zdd„ Zedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZdd„ Zd d!„ Zd"d#„ Zd$d%„ Z d&d'„ Z!d(d)„ Z"d*d+„ Z#d,d-„ Z$d.d/„ Z%e%Z&dS )0r»  rF   .Úconstant_args)Údefault_factoryrL   NÚoutput_viewc                 C   s    t | jtƒr|  ¡  |  ¡  d S rI   )r4   rd  rõ   Úapply_constraintrÛ  rŽ   rF   rF   rG   rr  o  s    zExternKernel.decide_layoutc                 C   s    t | |ƒ\}}|r| |¡ d S rI   )r)   Ú	writeline)r�   ÚwrapperZ
origin_strZdetailed_origin_strrF   rF   rG   Úcodegen_commentt  s    zExternKernel.codegen_commentc                 C   s
   t ƒ ‚d S rI   rá  ©r�   rÊ  rF   rF   rG   Úcodegeny  s    zExternKernel.codegenc                 C   s:   t j|  ¡ |  ¡ |  ¡ |  ¡ |  ¡ |  ¡ d�}| ¡  |S )N)r€   r   rÒ   rÓ   rÚ   r¯   )	r  rè   rˆ   r‡   r÷   r„   rä   r²   rÊ   )r‰   rÄ  rF   rF   rG   Ú
copy_input|  s    úzExternKernel.copy_inputc                    sn  t |ƒj||Žj}t|dd�\}}d }tjjrL|rLt|tj	j
ƒrLt|||ƒ}t |¡\}‰ g ‰g }	g }
|D ]R}ˆ t|tƒ¡ ˆd r’|	 |¡ qjt|tjƒr²tjjjj|d d�}|
 |¡ qj‡ ‡fdd„}‡fdd„|	D ƒ}	|	D ]}t|ƒrât|dd	� qâg }|	D ]B}| ¡ tjjk�r2| tjj| ¡  ¡ n| t|dd
�¡ �q|||
ƒ\}}|||Ž}||	|
||fS )NT)Zreturn_schemasrò  )r¢  c                    sd   g }t | ƒ}t |ƒ}ˆD ]&}|r0| t|ƒ¡ q| t|ƒ¡ qt |ˆ ¡}| dg ¡| di ¡fS )NrK   rL   )Úiterrp  ÚnextÚpytreeZtree_unflattenÚget)Znew_tensor_argsZnew_non_tensor_argsr\  Z
it_tensorsZit_non_tensorsZ	is_tensorrë   )Ú	args_specÚis_arg_tensorrF   rG   Úunflatten_args§  s    z3ExternKernel.process_kernel.<locals>.unflatten_argsc                    s   g | ]}ˆ   |¡‘qS rF   ©r¼  r‚  ©ré   rF   rG   rW   ³  s     z/ExternKernel.process_kernel.<locals>.<listcomp>rÏ  ©rŠ   )r   ÚbindÚ	argumentsr   r1   r�   rž   r4   r7   Ú_opsZOpOverloadPacketr/   rÑ  Ztree_flattenrp  r¨   r;   r   r‚   r  Zcreate_symintnoder…   rÑ  rè  Ú	constantsrŒ   )ré   rÃ  rK   rL   Zbinded_argsrv   ZschemasÚschemaZ	args_flatÚtensor_argsÚnon_tensor_argsÚargrÕ  r‰   Zexample_argsÚnew_argsZ
new_kwargsÚexample_outputrF   )rÓ  ré   rÔ  rG   Úprocess_kernel‰  sD    ÿþý
zExternKernel.process_kernelc           	   	   C   sÖ   t |tƒst‚t |tƒr|S | ¡  ¡  tj| ¡ dd�\}}|d }| 	¡ |ƒ}t
jj ||¡}t
jj ||¡}t
jj ||¡}t||ƒ| }||kr®t d|||¡ tƒ ‚t|jt| ¡ | ¡ | ¡ ||d�d�S )z 
        In order to pass this to an extern kernel we need a
        ReinterpretView not a View.  This allows us to avoid some
        unneeded copies.
        rë   r2   r   z@convert_to_reinterpret_view failed: stride=%s offset=%s index=%srF  ©re  rd  )r4   rà  r@   rÝ  rñ  rÛ  r   ry  r„   r‘  r1   r�   r‚   rz  Zstride_varsZ
offset_varr+   r¦  r¥  rÉ   re  r�  rˆ   r‡   )	ré   r‰   Z
index_argsr£  rm  rU   r  r÷  ÚexpectedrF   rF   rG   Úconvert_to_reinterpret_viewÎ  s@    
 ÿ
üûþz(ExternKernel.convert_to_reinterpret_viewc                 C   sü   |d krt ƒ S t|tjtjjjtfƒr.t|ƒS t|t	ƒrZt
j tj|j| ¡ | ¡ d�¡S t|tƒrh|S t|tƒr~|  |j¡S t|tƒrŒ|S t|tƒrÜ| ¡  t| ¡ ƒrÜt| ¡ jtƒsÜz|  |¡W S  tk
rÚ   Y nX t|tƒrò| ¡  |S |  |¡S )N)r   r€   ) r~  r4   r;   r   r=   r>   r?   rC  r€  rC  r1   r�   Zadd_tensor_constantr7   ZtensorrŸ   r‡   rˆ   r  r:   r¼  re  rÝ  rà  rÊ   r…   rñ  r  ræ  rÉ   rÙ  rÎ  ©ré   r‰   rF   rF   rG   r¼  û  s8    
ÿ



 ÿ
zExternKernel.realize_inputc                 C   sD   t |ƒr:t| ¡ ƒdkr|S | ¡ D ]}|dkr$|  S q$|  |¡S rö  )r…   rY   r1  rÎ  )ré   r‰   r~   rF   rF   rG   Úrequire_stride1  s    
zExternKernel.require_stride1c                 C   sr  |  ¡ dkr|S t|ƒr¬t| ¡ tƒr:t|dd|d� |S t| ¡ tƒrZ| ¡  |¡rZ|S t| ¡ tƒr¬t| ¡  	¡ tƒr„t
dƒ‚n(t| ¡  	¡ tƒr¬| ¡  	¡  |¡r¬|S t|tƒrÈ| ¡  |¡rÈ|S t|tƒ�rDt|jtƒ�rDt|jtƒ�sDt| ¡ ƒ�rDt| ¡ jtƒ�sDz|  |j¡|_|  ||¡W S  tk
�rB   Y nX |  |¡}t|dd|d� t||ƒ�snt
‚|S )Nr   TFrÖ  z<the MutationLayout's real layout shouldn't be FlexibleLayout)rÃ   r…   r4   r†   rõ   rÑ  r�  rÞ  ri  rl  r@   r|  r:   re  rà  rÝ  rñ  r  ræ  Úrequire_stride_orderrÉ   rÎ  rß  )ré   r‰   ra   rF   rF   rG   ré  $  sf       ÿ ÿþÿ
 ÿþÿ
þ
ý
üû
z!ExternKernel.require_stride_orderc                 C   s   |   |t¡S rI   )ré  ÚNHWC_STRIDE_ORDERrç  rF   rF   rG   Úrequire_channels_lastS  s    z"ExternKernel.require_channels_lastc              	   C   s    |   |tttt| ¡ ƒƒƒƒ¡S rI   )ré  r5   r*  rZ   rY   r„   rç  rF   rF   rG   Úrequire_contiguousW  s    zExternKernel.require_contiguousc                 C   s   d S rI   rF   rŽ   rF   rF   rG   rÈ  [  s    zExternKernel.apply_constraintc                 C   s   t tjjj| jƒS rI   )r¹   r1   r�   r7  Úval_to_arg_strrÅ  rŽ   rF   rF   rG   Úcodegen_const_args^  s    zExternKernel.codegen_const_argsc                 C   sf   g }| j D ]H}t|tƒrDdd„ |D ƒ}dd |¡› d�}| |¡ q
| | ¡ ¡ q
| |  ¡ ¡ |S )Nc                 S   s   g | ]}|  ¡ ‘qS rF   ©r9  rR   rF   rF   rG   rW   e  s     z-ExternKernel.codegen_args.<locals>.<listcomp>ú[r  ú])r­  r4   r5   r¸   rp  r9  r·  rî  )r�   rK   r‰   rÛ   r9  rF   rF   rG   Úcodegen_argsa  s    

zExternKernel.codegen_argsc                 C   sš   || j kr| j  |¡S t| dƒr†|| jkr†| j |¡ d¡}|d kr‚| j |¡ d¡}t|ƒtksttdt|ƒ› �ƒ‚tt|ƒ ƒ S |S td|› d�ƒ‚d S )NÚkwargs_default_valuerŸ   rA   ú$unsupported default_value arg_type: zarg z6 not found in self.kwargs or self.kwargs_default_value)rL   rÒ  r…  ró  rP   Údefault_value_mapr@   )r�   Úarg_namerª  Úarg_typerF   rF   rG   Úget_kwargs_valuem  s$    
ÿþ
ÿþ
ÿzExternKernel.get_kwargs_valuec                 C   sd   g }| j r`tjjrL| js tdƒ‚| jD ]"}|  |¡}| tjj 	|¡¡ q&ndd„ | j  
¡ D ƒ}|S )Nz0ordered_kwargs_for_cpp_kernel has to be providedc                 S   s(   g | ] \}}|› d t jj |¡› �‘qS rÖ   ©r1   r�   r7  rí  ©rS   Úkr‰  rF   rF   rG   rW   Ž  s   ÿz/ExternKernel.codegen_kwargs.<locals>.<listcomp>)rL   r1   r�   rž   Úordered_kwargs_for_cpp_kernelr@   rø  rp  r7  rí  r¢  )r�   rL   rö  r‰  rF   rF   rG   Úcodegen_kwargs‚  s    ÿþ

þzExternKernel.codegen_kwargsc              	   C   sX   t jrTtjjsTtjj |  ¡ ¡}tjj |  ¡ ¡}| 	d|  
¡ › d|› d|› d�¡ d S )Nzassert_size_stride(r  r6  )r   Zsize_assertsr1   r�   rž   r7  r8  r„   r1  rÉ  rè  )r�   rÊ  r}   r~   rF   rF   rG   Úcodegen_size_asserts”  s    ÿz!ExternKernel.codegen_size_assertsc                 C   s   |   ¡ }|  ¡ }|g g|fS )zD
        get output sizes and strides, for template_codegen
        )r„   r1  )r�   Ú_sizerH  rF   rF   rG   Úget_group_strideœ  s    zExternKernel.get_group_stridec                    s  t jj‰|  ¡ }|  ¡ }‡fdd„|D ƒ}dd„ tt|ƒƒD ƒ‰ttt|ƒƒ|jdd�}dd„ t	|ƒD ƒ‰‡fdd„ttˆƒƒD ƒ}‡fd	d„|D ƒ‰|  
¡ }|ˆƒ}t jj ˆ||g¡\}}}	td
ƒ\}
‰ ttˆ|‡ fdd„|D ƒƒƒƒ}tt |¡|ƒ}|t|ƒfS )zC
        Manually get canonicalization of the output index
        c                    s   g | ]}ˆ   |¡‘qS rF   )rƒ   r‚  )r‚   rF   rG   rW   ­  s     z-ExternKernel.canonicalize.<locals>.<listcomp>c                 S   s   g | ]}t d |› �ƒ‘qS )Údr  rR   rF   rF   rG   rW   ®  s     T)rN  Úreversec                 S   s   i | ]\}}||“qS rF   rF   rk   rF   rF   rG   rn   ±  s      z-ExternKernel.canonicalize.<locals>.<dictcomp>c                    s   g | ]}ˆ | ‘qS rF   rF   rR   ro   rF   rG   rW   ²  s     c                    s   g | ]}ˆ | ‘qS rF   rF   rR   rŒ  rF   rG   rW   ³  s     Úcc                    s   g | ]}ˆ |ƒ‘qS rF   rF   r‚  )Úadd_varrF   rG   rW   ¾  s     )r1   r�   r‚   r„   r1  rZ   rY   rn  r`  rq   r‘  r–  r#   r^   r_   r-   r;   r‰  r6   )r�   r^  r  Zindex_orderra   r  rU   Z	new_sizesr]   r™  rv   ÚreplacementrF   )r  r�  rp   r‚   rG   Úcanonicalize¥  s(      ÿ
 zExternKernel.canonicalizec                    sP   t ˆ dd ƒ}d|›�g}|‡ fdd„t ˆ ¡D ƒ7 }| dˆ j›�¡ ˆ  |¡S )NrÃ  zkernel=c                    s$   g | ]}|j › d tˆ |j ƒ› �‘qS rÖ   )rN   rJ   )rS   ÚfieldrŽ   rF   rG   rW   È  s   ÿz(ExternKernel.__str__.<locals>.<listcomp>rØ   )rJ   ÚdataclassesÚfieldsrp  rÚ   r¼   )r�   Zkernel_namer»   rF   rŽ   rG   rÜ   Ã  s    ÿþzExternKernel.__str__)'r’   r“   r”   rÅ  r   r   rË   r  r  r^   rL   r	   rP   rÇ  r   rÝ  rr  rË  rÍ  rÌ   rÎ  rú   rã  ræ  r¼  rè  ré  rë  rì  rÈ  rî  rò  rø  rý  rþ  r   r  rÜ   r‘   rF   rF   rF   rG   r»  i  s@   


D
,

	
.

	r»  c                       s@   e Zd ZU dZee ed< dd„ Zd
‡ fdd„	Zdd	„ Z	‡  Z
S )ÚExternKernelOutNrÇ  c                 C   s6   |   |¡ |  ¡ |  ¡ •}| | j|  ¡ || j¡ d S rI   )rË  rò  rý  Zgenerate_extern_kernel_outrÇ  r9  rÃ  ©r�   rÊ  rK   rF   rF   rG   rÍ  Ö  s    
üzExternKernelOut.codegenrF   c	           	         sN   t ƒ  d ||  |¡||pi ¡ || _tj | ¡| _tjjr>|n|| _	|| _
d S rI   )rÞ   r�   r¬  rÇ  r1   r�   r¯  rN   rž   rÃ  rü  )	r�   rd  r­  rÅ  rL   rÇ  rÃ  Ú
cpp_kernelrü  rß   rF   rG   r�   à  s        ÿzExternKernelOut.__init__c                 C   s   dS r¨  rF   rŽ   rF   rF   rG   r©  ó  s    zExternKernelOut.should_allocate)rF   NNNNrF   )r’   r“   r”   rÇ  r   rÝ  rË   rÍ  r�   r©  rû   rF   rF   rß   rG   r
  Ò  s   
      ÷r
  c                       s&   e Zd Zeejdœ‡ fdd„Z‡  ZS )ÚRandomSeeds)Úcountr€   c                    s@   t  t j¡}tƒ jt|t j|gd�g |j|j|ggddd� d S )Nrb  zaten.randint.low_outzat::randint_out)rd  r­  rÅ  rÃ  r  )r7   r§  r‹  rÞ   r�   r�  r!  r   )r�   r  r€   Zlimitsrß   rF   rG   r�   ø  s    ý÷zRandomSeeds.__init__)r’   r“   r”   rC  r7   r€   r�   rû   rF   rF   rß   rG   r  ÷  s   r  c                       s6   e Zd Zdd„ Zd‡ fdd„	Zdd„ Zd	d
„ Z‡  ZS )r  c                 C   sR   |   |¡ |  ¡ |  ¡ •}tjj |  ¡ | j||  	¡ ¡ t
| jtƒrN|  |¡ d S rI   )rË  rò  rý  r1   r�   r7  Zgenerate_extern_kernel_allocrè  rÃ  rä   r4   rd  r-  rþ  r  rF   rF   rG   rÍ    s    
   ÿzExternKernelAlloc.codegenrF   Nc                    sH   t ƒ  d ||  |¡||pi ¡ tj | ¡| _tjjr8|n|| _|| _	d S rI   )
rÞ   r�   r¬  r1   r�   r¯  rN   rž   rÃ  rü  )r�   rd  r­  rÅ  rL   rÃ  r  rü  rß   rF   rG   r�     s    
    ÿzExternKernelAlloc.__init__c                 C   s   dS rå   rF   rŽ   rF   rF   rG   r©  "  s    z!ExternKernelAlloc.should_allocatec                 C   s   t ‚d S rI   rá  rŽ   rF   rF   rG   rÈ  %  s    z"ExternKernelAlloc.apply_constraint)rF   NNNrF   )r’   r“   r”   rÍ  r�   r©  rÈ  rû   rF   rF   rß   rG   r    s        ør  c                       s<   e Zd ZdZdZdd„ Zdd„ Zdd„ Z‡ fd	d
„Z‡  Z	S )ÚInplaceBernoulliFallbackzE
    This needs to be a custom class to handle mutation properly
    zaten.bernoulli_c                 C   sB   dd„ | j D ƒ\}| | j› d|› dd tt| jƒ¡› d�¡ d S )Nc                 s   s   | ]}|  ¡ V  qd S rI   rï  ©rS   r‹   rF   rF   rG   rj  1  s     z3InplaceBernoulliFallback.codegen.<locals>.<genexpr>ú(r  r6  )r­  rÉ  rÃ  r¸   r¹   ÚreprrÅ  )r�   rÊ  r‰   rF   rF   rG   rÍ  0  s    $ÿz InplaceBernoulliFallback.codegenc                 C   s   dS rå   rF   rŽ   rF   rF   rG   r©  6  s    z(InplaceBernoulliFallback.should_allocatec                 C   s   t | jtƒst‚| jj ¡ fS rI   ©r4   rd  ri  r@   r   rè  rŽ   rF   rF   rG   r{  9  s    z+InplaceBernoulliFallback.get_mutation_namesc                    s0   t ƒ  d t|ƒ|  |g¡|¡ tj | ¡| _d S rI   )rÞ   r�   ri  r¬  r1   r�   r¯  rN   )r�   r‰   rÅ  rß   rF   rG   r�   =  s    
üz!InplaceBernoulliFallback.__init__)
r’   r“   r”   r:  rÃ  rÍ  r©  r{  r�   rû   rF   rF   rß   rG   r  )  s   r  c                       sN   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	d
œeee	 e
dœ‡ fdd„Z‡  ZS )ÚScatterFallbackzÅ
    This needs to be a custom class to handle mutation properly.
    This class handles both aten.scatter_ and aten.scatter_reduce_.
    It also handle the case `src` being a scalar properly.
    c              
   C   st   | j rdd„ | jD ƒ\}}}ndd„ | jD ƒ\}}| jd }| ||| jd ||g| j| j| j | jd |  ¡ ¡ d S )Nc                 s   s   | ]}|  ¡ V  qd S rI   rï  r  rF   rF   rG   rj  P  s     z*ScatterFallback.codegen.<locals>.<genexpr>c                 s   s   | ]}|  ¡ V  qd S rI   rï  r  rF   rF   rG   rj  R  s     r   r   r…  )Úsrc_is_tensorr­  rÅ  Zgenerate_scatter_fallbackrÃ  rO   rL   rý  )r�   rÊ  r‰   rU   rr  rF   rF   rG   rÍ  N  s    
ùzScatterFallback.codegenc                 C   s   dS rå   rF   rŽ   rF   rF   rG   r©  ^  s    zScatterFallback.should_allocatec                 C   sN   |dkr6| j r |d krdnd}qJ|d ks0tdƒ‚d}n|d k	sFtdƒ‚d}|S )Núaten.scatter_zat::scatter_outzat::scatter_reduce_outz:Expect reduce to be None for aten.scatter_ with scalar srcz5Expect reduce to be not None for aten.scatter_reduce_)r  r@   )r�   rO   r…  rÃ  rF   rF   rG   Úget_cpp_kernela  s    ÿÿþÿþzScatterFallback.get_cpp_kernelNT©r…  Úinclude_self)r  r…  r  c             	      sÚ   |dkst ‚t|tƒˆ _tjjrJdddœ}||kr:|| }ˆ  ||¡ˆ _n|ˆ _|ˆ _	ˆ jr|‡ fdd„|||fD ƒ}	|f}
n‡ fdd„||fD ƒ}	||f}
t
ƒ  d t|ƒˆ  |	¡|
||dœ¡ d	d
gˆ _tj ˆ ¡ˆ _d S )N>   úaten.scatter_reduce_r  r#  r"  )r  Úmultiplyc                    s   g | ]}ˆ   |¡‘qS rF   rÖ  r  rŽ   rF   rG   rW   �  s     z,ScatterFallback.__init__.<locals>.<listcomp>c                    s   g | ]}ˆ   |¡‘qS rF   rÖ  r  rŽ   rF   rG   rW   �  s     r  r…  r  )r@   r4   r:   r  r1   r�   rž   r  rÃ  rO   rÞ   r�   ri  r¬  rü  r¯  rN   )r�   rO   r‰   r  rU   rr  r…  r  Zget_operator_enumÚtensorsrÅ  rß   rŽ   rG   r�   s  s.    
û
zScatterFallback.__init__)r’   r“   r”   r:  rÍ  r©  r  rC  r   rP   r“  r�   rû   rF   rF   rß   rG   r  G  s   ÷÷r  c                       s0   e Zd ZdZdd„ Zdd„ Z‡ fdd„Z‡  ZS )ÚIndexPutFallbackzQ
    This needs to be a custom class to handle mutation and indices properly
    c                 C   s°   dd„ | j D ƒ^}}}g }t|ƒ}t| jƒD ]6\}}| j| d k	rR| t|ƒ¡ q,| tjjj	¡ q,tjjj
› d |¡› tjjj› �}	||	|f|  ¡ •}
| | | j|
¡¡ d S )Nc                 s   s   | ]}|  ¡ V  qd S rI   rï  r  rF   rF   rG   rj  £  s     z+IndexPutFallback.codegen.<locals>.<genexpr>r  )r­  rÏ  rq   ru  rp  rÐ  r1   r�   r7  r  Úopen_bracketr¸   Zclosed_bracketrî  rÉ  Zwrap_kernel_callrÃ  )r�   rÊ  r‰   rÅ  Úvalid_indicesru  Ziter_valid_indicesrT   rv   Zindices_strrK   rF   rF   rG   rÍ  ¢  s    "zIndexPutFallback.codegenc                 C   s   dS rå   rF   rŽ   rF   rF   rG   r©  °  s    z IndexPutFallback.should_allocatec                    sp   |ˆ _ dd„ |D ƒ}‡ fdd„||f|•D ƒ}tƒ  d t|ƒˆ  |¡|f¡ tj ˆ ¡ˆ _tjj	rfdndˆ _
d S )Nc                 S   s   g | ]}|d k	r|‘qS rI   rF   rR   rF   rF   rG   rW   µ  s      z-IndexPutFallback.__init__.<locals>.<listcomp>c                    s   g | ]}ˆ   |¡‘qS rF   rÖ  r‚  rŽ   rF   rG   rW   ¶  s     zat::index_put_zaten.index_put_)ru  rÞ   r�   ri  r¬  r1   r�   r¯  rN   rž   rÃ  )r�   r‰   ru  rÅ  Ú
accumulater  r  rß   rŽ   rG   r�   ³  s    üzIndexPutFallback.__init__)r’   r“   r”   r:  rÍ  r©  r�   rû   rF   rF   rß   rG   r  �  s   r  c                   @   s    e Zd Zedd„ ƒZdd„ ZdS )Ú
DeviceCopyc                 C   s˜   |  ¡ s(tdd„ | ¡ D ƒƒr(| |¡S tjj |j¡ tj 	|j
¡ tjj | ¡ j¡ tj 	| ¡ j
¡ tdƒ tt|| ¡ | ¡ d�|  |¡gƒS )Nc                 s   s(   | ] }|j tjjkot|tjƒV  qd S rI   )rN   r1   r�   rÜ  r4   r   r‡  rg  rF   rF   rG   rj  Ä  s   ÿz$DeviceCopy.create.<locals>.<genexpr>zDeviceCopy in input programrb  )ræ   ro  rÂ   r  r1   r�   Zdevice_typesr  rA   Zadd_device_idxrU   rˆ   r(   r!  rõ   r‡   r„   r¼  )ré   r‰   r€   rF   rF   rG   rè   Â  s"    þ
ý
úzDeviceCopy.createc                 C   sv   |   ¡ }t|ƒdkst‚| jrJ| | j ¡ › d|d › dtjjj	› �¡ n(| |  ¡ › d|d › dtjjj	› �¡ d S )Nr   ú.copy_(r   r6  )
rò  rY   r@   rÇ  rÉ  r9  r1   r�   r7  Úendingr  rF   rF   rG   rÍ  Ù  s    "ÿ ÿzDeviceCopy.codegenN)r’   r“   r”   rú   rè   rÍ  rF   rF   rF   rG   r!  Á  s   
r!  c                   @   s   e Zd ZdZdd„ ZdS )r9   zó
    The result of a call to aten._local_scalar_dense.

    This is not yet implemented.  The one model (so far) that calls this
    (fastNLP_Bert) does not actually use the result.  So we expect this
    node to get dead code eliminated.
    c                 C   s   dS rB  rF   rŽ   rF   rF   rG   rÂ   ï  s    zDynamicScalar.get_readsN)r’   r“   r”   r:  rÂ   rF   rF   rF   rG   r9   æ  s   r9   c                       sf   e Zd Zd‡ fdd„	Zdd„ Zdd„ Zdd	„ Zed
d„ ƒZ‡ fdd„Z	e
dd„ ƒZ‡ fdd„Z‡  ZS )ÚFallbackKernelNc           	         sP  t ƒ  |t|ƒt|ƒ¡ d| _t|tjjƒr2|jn|}t	tj
j|jd ƒ|kr®tjjr`d|j› �n
d|j› �| _|d k	r¬dd„ |jD ƒ| _dd„ |jD ƒ| _dd„ |jD ƒ| _nvt|tjjƒrìt	tjj|jd ƒ|krâd	|j› �| _ntd
ƒ‚n8tjj�rd| _|  |¡ n|j dd¡› d|j› �| _|| _|d k�r8i n|| _tj | j¡ d S )NFzat::zaten.c                 S   s    g | ]}|j s|j|jd œ‘qS ©)rA   rŸ   )Ú
kwarg_onlyÚ	real_typerª  r‚  rF   rF   rG   rW     s   þz+FallbackKernel.__init__.<locals>.<listcomp>c                 S   s   g | ]}|j r|j‘qS rF   ©r&  rN   r‚  rF   rF   rG   rW     s     c                 S   s$   i | ]}|j r|j|j|jd œ“qS r%  )r&  rN   r'  rª  r‚  rF   rF   rG   rn     s   þ z+FallbackKernel.__init__.<locals>.<dictcomp>ztorch._prims.rng_prims.z.Unable to find HigherOrderOperator kernel nameTz._ops.z.ops.Ú.)rÞ   r�   r6   Úuse_cpp_op_schemar4   r7   rÛ  Z
OpOverloadZ_overloadpacketrJ   r0   Úatenr’   r1   r�   rž   rÃ  rÚ  Úargs_default_valuerü  ró  ZHigherOrderOperatorZ_primsZ	rng_primsrÉ   Úset_cpp_kernelr“   ÚreplacerÕ  rL   Zwarn_fallback)	r�   rd  rÃ  rÞ  Znontensor_argsrÕ  rL   rÝ  Zop_overload_packetrß   rF   rG   r�   õ  sT    
ýÿýÿÿÿ
ýþÿþ
ÿ
ÿzFallbackKernel.__init__c                    sÔ   ddl m} |jjr&td|j› d�ƒ‚dd„ ‰ t‡ fdd„|jjD ƒƒsXt|j› d	�ƒ‚t‡ fd
d„|jjD ƒƒs‚t|j› d�ƒ‚|jj	| _
|jj| _| j
 dd¡› d| j› �| _||ƒ| _dd„ |jjD ƒ| _d S )Nr   )Úget_cpp_op_schemazmutable z" is not supported with cpp_wrapperc                 S   s   | j d kp| j j S rI   )Z
alias_infoZis_write)rà  rF   rF   rG   Úis_not_write@  s    z3FallbackKernel.set_cpp_kernel.<locals>.is_not_writec                 3   s   | ]}ˆ |ƒV  qd S rI   rF   r‚  ©r0  rF   rG   rj  C  s    z0FallbackKernel.set_cpp_kernel.<locals>.<genexpr>z< with alias_info arguments is not supported with cpp_wrapperc                 3   s   | ]}ˆ |ƒV  qd S rI   rF   r‚  r1  rF   rG   rj  F  s    z: with alias_info returns is not supported with cpp_wrapperz::rv   c                 S   s   g | ]}|j r|j‘qS rF   r(  r‚  rF   rF   rG   rW   Q  s     z1FallbackKernel.set_cpp_kernel.<locals>.<listcomp>)Zcodegen.wrapperr/  Z_schemaZ
is_mutabler@   r’   ro  rÚ  ZreturnsrN   rÃ  Zoverload_nameÚcpp_kernel_overlad_namer.  Úcpp_kernel_keyÚcpp_op_schemarü  )r�   rÃ  r/  rF   r1  rG   r-  6  s0    ÿþÿ

þÿ

þ

ÿ
ÿzFallbackKernel.set_cpp_kernelc                 C   s’   t | dƒstdƒ‚|t| jƒk s:td|› dt| jƒ› �ƒ‚| j| d }|d krŠ| j| d }t|ƒtks|tdt|ƒ› �ƒ‚tt|ƒ ƒ S |S d S )Nr,  z*self.args_default_value has to be providedzexpected the index z2 to be smaller than len(self.args_default_value): rŸ   rA   rô  )r…  r@   rY   r,  rP   rõ  )r�   rm   r‰  r÷  rF   rF   rG   Úget_arg_default_valueU  s(     ÿþÿþ
ÿþz$FallbackKernel.get_arg_default_valuec                    s´   t jG dd„ dƒƒ‰ ‡ fdd„ˆjD ƒ}ˆ |ˆj¡\}}dd„ |D ƒ}tjjr¤tˆdƒr¤t	|ƒ}t	ˆj
ƒ}||k r¤‡fdd„t||ƒD ƒ}dd„ |D ƒ}| |¡ ˆj |¡ |S )	Nc                   @   s   e Zd ZU eed< dd„ ZdS )z)FallbackKernel.codegen_args.<locals>.ShimÚrefc                 S   s   | j S rI   )r6  rŽ   rF   rF   rG   r‘   l  s    z2FallbackKernel.codegen_args.<locals>.Shim.__repr__N)r’   r“   r”   r   rË   r‘   rF   rF   rF   rG   ÚShimh  s   
r7  c                    s   g | ]}ˆ |  ¡ ƒ‘qS rF   rï  r‚  )r7  rF   rG   rW   o  s     z/FallbackKernel.codegen_args.<locals>.<listcomp>c                 S   s   g | ]}t jj |¡‘qS rF   rù  r‚  rF   rF   rG   rW   q  s     r,  c                    s   g | ]}ˆ   |¡‘qS rF   )r5  rR   rŽ   rF   rG   rW   w  s    c                 S   s   g | ]}t jj |¡‘qS rF   rù  r‚  rF   rF   rG   rW   z  s     )r  Ú	dataclassr­  rÕ  rÅ  r1   r�   rž   r…  rY   r,  rZ   r·  rL   Úupdate)r�   rÞ  rK   rL   Zn_argsZ
n_pos_argsZpos_argsrF   )r7  r�   rG   rò  g  s     

ÿ
zFallbackKernel.codegen_argsc                 C   sˆ   | r| d   ¡ S t|tjƒr"|jS t|ttfƒr„dd„ |D ƒ}dd„ |D ƒ}t|ƒdkr`|d S |D ]}|jdkrd|  S qd|d S d S )Nr   c                 S   s   h | ]}t  d |¡’qS rI   )r$  Úfind_devicer‚  rF   rF   rG   rÁ   ˆ  s     z-FallbackKernel.find_device.<locals>.<setcomp>c                 S   s   g | ]}|r|‘qS rF   rF   )rS   r€   rF   rF   rG   rW   Š  s      z.FallbackKernel.find_device.<locals>.<listcomp>r   r£   )	rˆ   r4   r7   ÚTensorr€   r5   r6   rY   rA   )rÞ  râ  Zdevicesr€   rF   rF   rG   r:  �  s    

zFallbackKernel.find_devicec                    sH   | j r8|  ¡ |  ¡ •}| |  ¡ | j|| j| j| j¡ nt	ƒ  
|¡ d S rI   )r*  rò  rý  Ú6generate_extern_kernel_alloc_and_find_schema_if_neededrè  rÃ  r4  r3  r2  rÞ   rÍ  r  rß   rF   rG   rÍ  “  s    ú	zFallbackKernel.codegenc              	      s’   t jf}||krtjjntƒ }|�" | j|f|ž|Ž\}}}}	}
W 5 Q R X t ||¡}|sbt	dƒ‚tt
|ƒ||||	|
d�‰‡ ‡fdd„‰ ˆ |g ƒS )Nz"Not sure where to find device info)rÝ  c                    s�   t ˆttfƒr4tˆƒ‡‡ ‡fdd„ttˆƒƒD ƒƒS t ˆtjƒrjtt	ˆj
ˆjtˆ ¡ ƒtˆ ¡ ƒƒˆˆ ƒS t ˆtƒrxˆS ˆd ksˆtdƒ‚d S d S )Nc                 3   s*   | ]"}ˆ ˆ| ˆt ˆƒ|fg ƒV  qd S rI   )rA   rR   )Úgenerate_outputru  ÚoutputrF   rG   rj  ½  s   ÿzAFallbackKernel.create.<locals>.generate_output.<locals>.<genexpr>z+FallbackKernel output type is not supported)r4   r5   r6   rA   rZ   rY   r7   r;  ÚMultiOutputr�  r€   r   r&   r}   r~   rC  r@   )r>  ru  ©r=  Úpacked)ru  r>  rG   r=  »  s&    
þ

üø

z.FallbackKernel.create.<locals>.generate_output)r+  Z*_fused_moving_avg_obs_fq_helper_functionalr1   r�   Ú	fake_moder   rã  r$  r:  r@   rv  )ré   rÃ  rK   rL   Zfake_incorrect_kernelsÚcontextrâ  rÞ  rß  rÕ  rÝ  r€   rF   r@  rG   rè   ¡  s0    ÿúû
ú	zFallbackKernel.createc                    s
   t ƒ  ¡ S rI   )rÞ   rÈ  rŽ   rß   rF   rG   rÈ  Ô  s    zFallbackKernel.apply_constraint)NN)r’   r“   r”   r�   r-  r5  rò  rÌ   r:  rÍ  rú   rè   rÈ  rû   rF   rF   rß   rG   r$  ó  s   	  øA

2r$  c                   @   s   e Zd ZU ejed< dS )rv  r€   N)r’   r“   r”   r7   r€   rË   rF   rF   rF   rG   rv  Ø  s   
rv  c                       sF   e Zd Zdd„ Zdd„ Zeeedf  dœ‡ fdd„Zd	d
„ Z	‡  Z
S )r?  c                 C   s‚   t |ƒdkrz|d \}}|tkr@|  |› d|› d�|dd … ¡S |tkrptjj |t|ƒ¡}|  ||dd … ¡S t	dƒ‚n|S d S )Nr   rð  rñ  r   znon supported index type)
rY   r5   Úcodegen_list_tuple_accessr6   r1   r�   r7  Zcodegen_tuple_accessrP   r@   )r�   Úbasenameru  ZityperT   Ztuple_accessrF   rF   rG   rD  Þ  s      ÿ
z%MultiOutput.codegen_list_tuple_accessc                 C   sb   t jjj}||  ¡ › d|  | jd  ¡ | j¡› �7 }|t jjj7 }t jj 	|¡ |  
t jj¡ d S ©Nú = r   )r1   r�   r7  Zdeclarerè  rD  r­  ru  r#  rÉ  rþ  )r�   rÊ  ÚlinerF   rF   rG   rÍ  î  s
    
*zMultiOutput.codegen.)ru  c                    s,   t ƒ  d ||gd¡ tj | ¡| _|| _d S rB  )rÞ   r�   r1   r�   r¯  rN   ru  )r�   rd  rº  ru  rß   rF   rG   r�   õ  s    zMultiOutput.__init__c                 C   s   dS rå   rF   rŽ   rF   rF   rG   r©  ú  s    zMultiOutput.should_allocate)r’   r“   r”   rD  rÍ  r
   r   r   r�   r©  rû   rF   rF   rß   rG   r?  Ý  s   r?  r:   )	r‰   rË  ÚbiasÚpaddingr~   ÚdilationÚgroupsÚ
transposedÚoutput_paddingc
                 C   s~  dd„ }
dd„ }|  ¡  |  ¡  |dk	r0|  ¡  tjj��œ t|dd�}t|dd�}t| ¡ ƒd }d	t|ƒ  k r||ks‚n t‚d	t|ƒ  k rš|ks n t‚d	t|ƒ  k r¸|ks¾n t‚t||ƒ}t||ƒ}t||ƒ}|	dkròtd	g|ƒ}	n,d	t|	ƒ  k �r|k�sn t‚t|	|ƒ}	t	|t
ƒ�s.t‚|�r\|||ƒ}| ¡ }|
||||	|||ƒ}n@|dk	�rrt|dd�n|}tjj ||||||||	|¡	}| ¡ }d	gtttd
t|ƒd
 ƒƒƒ }t|ƒg| }t|ƒ}W 5 Q R X |  ||¡}| ¡ jdk�r| ¡ jdk�st‚||g}t| ¡ | ¡ t|ƒt|ƒƒ}||||g}|�rP| d
|	¡ |dk	�rf| |¡ n| d	|¡ ||||fS )au  
    This function is a helper function to prepare inputs, layout and constant args
    for convolution post-op fusion's create function, including deciding the output
    layout (channels first or channels last), realizing inputs and make them etc. The
    function only supports the CPU device since conv post-op fusion kernel is only
    supported on CPU right now.
    c                 S   sØ   t | ƒt |ƒkstdƒ‚t | ƒ}|dks0tdƒ‚d}d}	g }
|
 | | ¡ |
 ||	 | ¡ td|ƒD ]b}|| d ||d   d }| | d ||d   ||d  d  | ||d   }|
 |¡ qfttt|
ƒƒS )NzExpect input dim == weight dimrj   zExpect input dim > 2r   r   )rY   r@   rp  rZ   r5   r¹   rC  )Úoutput_sizeÚweight_sizerJ  rN  r~   rK  rL  r  Z	BATCH_DIMZWEIGHT_INPUT_CHANNELS_DIMrÁ  r  rÃ  Zinput_size_drF   rF   rG   Ú_conv_input_size  s(    ÿþ
ýÿz<_prepare_convolution_fusion_create.<locals>._conv_input_sizec                 S   s„   |   ¡ }t|ƒ}|dks tdƒ‚|dkrpg }| |d | ¡ | |d | ¡ td|ƒD ]}| || ¡ qZn|  dd¡  ¡ }|S )Nrj   zExpect weight dim > 2r   r   )r}   rY   r@   rp  rZ   Z	transpose)Zprepacked_weightrL  Zprepacked_weight_sizer  rP  r  rF   rF   rG   Ú_original_deconv_weight_size.  s    zH_prepare_convolution_fusion_create.<locals>._original_deconv_weight_sizeNTrØ  rj   r   r   r¦   )rÊ   r1   r�   rB  rŒ   rY   r}   r@   r*   r4   rC  r7   r0   r+  Zconvolutionr5   r*  rZ   r   ré  rˆ   rA   r�  r‡   r&   Úinsertrp  )ré   r‰   rË  rI  rJ  r~   rK  rL  rM  rN  rQ  rR  Úx_fakeÚweight_faker  rP  rÁ  rO  Ú	bias_faker>  Úreq_stride_orderrÂ  r­  Úkernel_layoutrÅ  rF   rF   rG   Ú"_prepare_convolution_fusion_createþ  s„    


"

ùÿ÷ $ü
rY  )r‰   rË  rI  c              	   C   s0  |  ¡  |  ¡  |dk	r |  ¡  tjj�~ t|dd�}t|dd�}|dk	rVt|dd�n|}|dk	rxtjjj 	|||¡}ntjjj
 	||¡}| ¡ }ddg}	t|ƒ}
W 5 Q R X |  ||	¡}| ¡ jdkrÔ| ¡ jdksØt‚||g}t| ¡ | ¡ t|ƒt|
ƒƒ}g }|dk	�r| |¡ n| d|¡ ||||	fS )zþ
    This function is a helper function to prepare inputs, layout and constant args
    for linear post-op fusion's create function. The function only supports the CPU device
    since linear post-op fusion kernel is only supported on CPU right now.
    NTrØ  r   r   r¦   )rÊ   r1   r�   rB  rŒ   r7   r0   r+  Zaddmmr³  Úmmr}   r   ré  rˆ   rA   r@   r�  r‡   r&   rp  rS  )ré   r‰   rË  rI  rT  rU  rV  r>  rO  rW  rÂ  r­  rX  rÅ  rF   rF   rG   Ú_prepare_linear_fusion_createŒ  sH    
ÿ
ý
þ ü
r[  c                	       sP   e Zd Zd‡ fdd„	Zdd„ Zedddee ee ee edœd	d
„ƒZ‡  Z	S )ÚConvolutionUnaryrF   ú'torch.ops.mkldnn._convolution_pointwisec                    s(   t ƒ j|||d ddd� d| _d| _d S )Nr]  úmkldnn::_convolution_pointwise©rÃ  r  Zconvolution_pointwiseaç  
            at::Tensor(
                const at::Tensor& input_t,
                const at::Tensor& weight_t,
                const c10::optional<at::Tensor>& bias_opt,
                at::IntArrayRef padding,
                at::IntArrayRef stride,
                at::IntArrayRef dilation,
                int64_t groups,
                c10::string_view attr,
                torch::List<c10::optional<at::Scalar>> scalars,
                c10::optional<c10::string_view> algorithm)©rÞ   r�   r3  r4  ©r�   rd  r­  rÅ  rÃ  rß   rF   rG   r�   Å  s    úzConvolutionUnary.__init__c                 C   s:   |  |  ¡ | j|  ¡ | j| j¡ t| jtƒr6|  	|¡ d S rI   )
r<  rè  rÃ  rò  r4  r3  r4   rd  r-  rþ  rÌ  rF   rF   rG   rÍ  â  s    ûzConvolutionUnary.codegenr:   )r‰   rË  rI  Úpadding_Ústride_Ú	dilation_rL  c              	   C   sR   t | |||||||ƒ\}}}}tƒ }tƒ }||t||	ƒt||
ƒg }t|||d�S ©N©rd  r­  rÅ  )rY  r•   r˜   r    r\  )ré   r‰   rË  rI  rb  rc  rd  rL  ÚattrÚscalarsÚ	algorithmr­  rÅ  rX  rv   r–   r™   rF   rF   rG   rè   í  s,           ÿýýzConvolutionUnary.create)rF   r]  ©
r’   r“   r”   r�   rÍ  rú   r
   rC  rè   rû   rF   rF   rß   rG   r\  Ä  s     ûør\  c                       sp   e Zd Zd
‡ fdd„	Zdd„ Zeddddee ee ee eee	e
 e	e e	ee  e	e dœdd	„ƒZ‡  ZS )ÚConvolutionBinaryrF   c                    s4   t ƒ j|||d ddd� d| _d| _d| _|| _d S )Nz.torch.ops.mkldnn._convolution_pointwise.binaryr^  r_  ÚbinaryZconvolution_pointwise_binarya’  
            at::Tensor(
                const at::Tensor& input_t,
                const at::Tensor& other_t,
                const at::Tensor& weight_t,
                const c10::optional<at::Tensor>& bias_opt,
                at::IntArrayRef padding,
                at::IntArrayRef stride,
                at::IntArrayRef dilation,
                int64_t groups,
                c10::string_view binary_attr,
                c10::optional<at::Scalar> alpha,
                c10::optional<c10::string_view> unary_attr,
                torch::List<c10::optional<at::Scalar>> unary_scalars,
                c10::optional<c10::string_view> unary_algorithm))rÞ   r�   r2  r3  r4  Úcpp_constant_args)r�   rd  r­  rÅ  rm  rß   rF   rG   r�     s    úzConvolutionBinary.__init__c                 C   s>   |  |  ¡ | j|  ¡ | j| j| j¡ t| jt	ƒr:|  
|¡ d S rI   )r<  rè  rÃ  rò  r4  r3  r2  r4   rd  r-  rþ  rÌ  rF   rF   rG   rÍ  /  s    úzConvolutionBinary.codegenr:   ©r‰   rX  rË  rI  rb  rc  rd  rL  Úbinary_attrÚbinary_alphaÚ
unary_attrÚunary_scalarsÚunary_algorithmc              	   C   s€   t | |||||||ƒ\}}}}|  ||¡}| d|¡ tƒ }tƒ }tƒ }||	t||
ƒt||ƒt||ƒt||ƒg }t|||d�S )Nr   rf  )rY  ré  rS  rš   r•   r˜   r    rk  )ré   r‰   rX  rË  rI  rb  rc  rd  rL  ro  rp  rq  rr  rs  r­  rÅ  rX  rW  r›   r–   r™   rF   rF   rG   rè   ;  s@           ÿûûýzConvolutionBinary.create)rF   rF   )r’   r“   r”   r�   rÍ  rú   r
   rC  rP   r   rÿ   r   rè   rû   rF   rF   rß   rG   rk    s&     û"
òrk  c                       sx   e Zd Zd‡ fdd„	Zdd„ Zdd„ Zeddddee ee ee ee	e
e e
e	 e
ee  e
e	 d	œd
d„ƒZ‡  ZS )ÚConvolutionBinaryInplacerF   c                    sJ   |d |d g|dd …  }t ƒ j|||d ddd� d| _d| _d	| _d S )
Nr   r   rj   z/torch.ops.mkldnn._convolution_pointwise_.binaryzmkldnn::_convolution_pointwise_r_  rl  Zconvolution_pointwise_binary_a�  
            at::Tensor&(
                at::Tensor& other_t,
                const at::Tensor& input_t,
                const at::Tensor& weight_t,
                const c10::optional<at::Tensor>& bias_opt,
                at::IntArrayRef padding,
                at::IntArrayRef stride,
                at::IntArrayRef dilation,
                int64_t groups,
                c10::string_view binary_attr,
                c10::optional<at::Scalar> alpha,
                c10::optional<c10::string_view> unary_attr,
                torch::List<c10::optional<at::Scalar>> unary_scalars,
                c10::optional<c10::string_view> unary_algorithm)©rÞ   r�   r2  r3  r4  )r�   rX  r­  rÅ  Zreordered_inputsrß   rF   rG   r�   h  s    úz!ConvolutionBinaryInplace.__init__c                 C   s(   |  |  ¡ | j|  ¡ | j| j| j¡ d S rI   ©r<  rè  rÃ  rò  r4  r3  r2  rÌ  rF   rF   rG   rÍ  Œ  s    úz ConvolutionBinaryInplace.codegenc                 C   s   t | jtƒst‚| jj ¡ fS rI   r  rŽ   rF   rF   rG   r{  –  s    z+ConvolutionBinaryInplace.get_mutation_namesr:   rn  c              	   C   sˆ   t | |||||||ƒ\}}}}|  ||¡}| d|¡ tƒ }tƒ }tƒ }||	t||
ƒt||ƒt||ƒt||ƒg }tt|d ƒ||d�S )Nr   )rX  r­  rÅ  )	rY  ré  rS  rš   r•   r˜   r    rt  ri  )ré   r‰   rX  rË  rI  rb  rc  rd  rL  ro  rp  rq  rr  rs  r­  rÅ  rv   rW  r›   r–   r™   rF   rF   rG   rè   š  s@           ÿûû
ýzConvolutionBinaryInplace.create)rF   )r’   r“   r”   r�   rÍ  r{  rú   r
   rC  rP   r   rÿ   r   rè   rû   rF   rF   rß   rG   rt  g  s&    ü$

òrt  c                       s2   e Zd Zd‡ fdd„	Zdd„ Zedd„ ƒZ‡  ZS )	ÚMKLPackedLinearrF   c                    s(   t ƒ j|||d ddd� d| _d| _d S )Nztorch.ops.mkl._mkl_linearzmkl::_mkl_linearr_  Z
mkl_lineara  
            at::Tensor(
                const at::Tensor& self,
                const at::Tensor& mkl_weight_t,
                const at::Tensor& origin_weight_t,
                const c10::optional<at::Tensor>& bias_opt,
                const int64_t prepack_batch_size)r`  ©r�   rd  r­  rÅ  rß   rF   rG   r�   Ç  s    úzMKLPackedLinear.__init__c                 C   s$   |  |  ¡ | j|  ¡ | j| j¡ d S rI   ©r<  rè  rÃ  rò  r4  r3  rÌ  rF   rF   rG   rÍ  Þ  s    ûzMKLPackedLinear.codegenc                 C   s‚   |   |  |¡¡}|   |  |¡¡}| ¡ �^ }}| ¡ \}}t|ƒ|g }t|ƒ}	|||g}
d |g}tt| ¡ | ¡ ||	ƒ|
|d�S re  )	rè  r¼  r„   r5   r   rw  r�  rˆ   r‡   )ré   r‰   Zpacked_wZorig_wZ
batch_sizeÚmrv   ÚocrO  rÂ  r­  rÅ  rF   rF   rG   rè   ç  s$    
   ÿûzMKLPackedLinear.create)rF   )r’   r“   r”   r�   rÍ  rú   rè   rû   rF   rF   rß   rG   rw  Æ  s
    ü	rw  c                       s:   e Zd Zd
‡ fdd„	Zdd„ Zedd„ ƒZdd	„ Z‡  ZS )ÚLinearUnaryrF   c                    s(   t ƒ j|||d ddd� d| _d| _d S )Nz"torch.ops.mkldnn._linear_pointwiseúmkldnn::_linear_pointwiser_  Zlinear_pointwiseaL  
            at::Tensor(
                const at::Tensor& input_t,
                const at::Tensor& weight_t,
                const c10::optional<at::Tensor>& bias_opt,
                c10::string_view attr,
                torch::List<c10::optional<at::Scalar>> scalars,
                c10::optional<c10::string_view> algorithm)r`  rx  rß   rF   rG   r�   ü  s    úzLinearUnary.__init__c                 C   s$   |  |  ¡ | j|  ¡ | j| j¡ d S rI   ry  rÌ  rF   rF   rG   rÍ    s    ûzLinearUnary.codegenc                 C   s°   |   |  |¡¡}|   |  |¡¡}| ¡ �^ }}| ¡ \}	}||g}
||rL|ndg|g}|d k	rz|   |  |¡¡}|
 |¡ n| dd ¡ tt| ¡ | ¡ t	|ƒ|	g d�|
|d�S )Nrò  r   rb  rf  )
rè  r¼  r„   rp  rS  r|  rõ   rˆ   r‡   r5   )ré   r‰   Úwr/  rg  rh  ri  rz  Úicr{  r­  rÅ  rF   rF   rG   rè     s&    ýùzLinearUnary.createc                 C   s   d S rI   rF   rŽ   rF   rF   rG   rÈ  7  s    zLinearUnary.apply_constraint)rF   )	r’   r“   r”   r�   rÍ  rú   rè   rÈ  rû   rF   rF   rß   rG   r|  û  s    ü	
r|  c                       s>   e Zd ZdZd‡ fdd„	Zdd„ Zedd„ ƒZd	d
„ Z‡  Z	S )ÚLinearBinaryú)torch.ops.mkldnn._linear_pointwise.binaryrF   c                    s.   t ƒ j|||d ddd� d| _d| _d| _d S )Nr�  r}  r_  rl  Zlinear_pointwise_binarya  
            at::Tensor(
                const at::Tensor& input_t,
                const at::Tensor& other_t,
                const at::Tensor& weight_t,
                const c10::optional<at::Tensor>& bias_opt,
                c10::string_view attr)
        ru  rx  rß   rF   rG   r�   >  s    úzLinearBinary.__init__c                 C   s(   |  |  ¡ | j|  ¡ | j| j| j¡ d S rI   rv  rÌ  rF   rF   rG   rÍ  W  s    úzLinearBinary.codegenc                 C   s´   |   |  |¡¡}|   |  |¡¡}|   |  |¡¡}| ¡ �^ }}| ¡ \}}|||g}	|g}
|d k	r~|   |  |¡¡}|	 |¡ n|
 d|¡ tt| ¡ | ¡ t	|ƒ|g d�|	|
d�S )Nr   rb  rf  )
rè  r¼  r„   rp  rS  r€  rõ   rˆ   r‡   r5   )ré   r‰   Úyr~  r/  rg  rz  r  r{  r­  rÅ  rF   rF   rG   rè   a  s(    
ýùzLinearBinary.createc                 C   s   d S rI   rF   rŽ   rF   rF   rG   rÈ  |  s    zLinearBinary.apply_constraint)rF   )
r’   r“   r”   rÃ  r�   rÍ  rú   rè   rÈ  rû   rF   rF   rß   rG   r€  ;  s    ü

r€  c                
       sV   e Zd Zd
‡ fdd„	Zdd„ Zedddee ee ee ee edœdd	„ƒZ‡  Z	S )ÚConvolutionTransposeUnaryrF   c                    s(   t ƒ j|||d ddd� d| _d| _d S )Nz1torch.ops.mkldnn._convolution_transpose_pointwisez(mkldnn::_convolution_transpose_pointwiser_  Zconvolution_transpose_pointwisea  
            at::Tensor(
                const at::Tensor& input_t,
                const at::Tensor& weight_t,
                const c10::optional<at::Tensor>& bias_opt,
                at::IntArrayRef padding,
                at::IntArrayRef output_padding,
                at::IntArrayRef stride,
                at::IntArrayRef dilation,
                int64_t groups,
                c10::string_view attr,
                torch::List<c10::optional<at::Scalar>> scalars,
                c10::optional<c10::string_view> algorithm)r`  rx  rß   rF   rG   r�   �  s    úz"ConvolutionTransposeUnary.__init__c                 C   s$   |  |  ¡ | j|  ¡ | j| j¡ d S rI   ry  rÌ  rF   rF   rG   rÍ  ž  s    ûz!ConvolutionTransposeUnary.codegenr:   )r‰   rË  rI  rb  Úoutput_padding_rc  rd  Úgroups_c                 C   sZ   d}t | |||||||||ƒ
\}}}}tƒ }tƒ }||	t||
ƒt||ƒg }t|||d�S )NTrf  )rY  r•   r˜   r    rƒ  )ré   r‰   rË  rI  rb  r„  rc  rd  r…  rg  rh  ri  rM  r­  rÅ  rX  rv   r–   r™   rF   rF   rG   rè   §  s<    öûýýz ConvolutionTransposeUnary.create)rF   rj  rF   rF   rß   rG   rƒ  €  s    ü	÷rƒ  c                       sZ   e Zd Zd‡ fdd„	Zdd„ Zedddddddeee eeeeeeedœd	d
„ƒZ	‡  Z
S )ÚMkldnnRnnLayerrF   úaten.mkldnn_rnn_layerc                    s   t ƒ  |||¡ || _d S rI   )rÞ   r�   rÃ  ra  rß   rF   rG   r�   ×  s    ýzMkldnnRnnLayer.__init__c              
   C   s0   |  |  ¡ › d| j› dd |  ¡ ¡› d�¡ d S )NrG  r  r  r6  )rÉ  rè  rÃ  r¸   rò  rÌ  rF   rF   rG   rÍ  á  s    $ÿzMkldnnRnnLayer.codegenr:   )r‰   Úw0Úw1Úw2Úw3ÚhxÚcxr  Úbatch_sizesr  Úhidden_sizeÚ
num_layersÚ
has_biasesÚbidirectionalÚbatch_firstÚtrainc              	      s\  |   |  ˆ¡¡‰ˆ ¡  |   |  |¡¡}|   |  |¡¡}|   |  |¡¡}|   |  |¡¡}|   |  |¡¡}| ¡  |   |  |¡¡}| ¡  ˆ ¡ }t|ƒdks¤tdƒ‚|\}}}|||g}| ¡ }| ¡ }g }ˆ||||||g}||	|
||||||g	}ttˆ ¡ ƒ||d�‰dd„ }g ‰ |||g}|||ƒt	|ƒt	|ƒg}‡ ‡‡fdd„t
t||ƒƒD ƒ}|S )Nri   zExpect lstm input to be 3D)r­  rÅ  c                 S   s   t | ƒdkstdƒ‚t| ƒS )Nri   zExpect output_shape to be 3D)rY   r@   r   )Úoutput_shaper“  rF   rF   rG   Úget_strides_of_lstm_output&  s    z9MkldnnRnnLayer.create.<locals>.get_strides_of_lstm_outputc                    s<   g | ]4\}\}}t tˆ ¡ ˆ ¡ ||ƒˆˆ t|fg ƒ‘qS rF   )r?  r�  rˆ   r‡   r5   )rS   rT   rO  rÂ  ©ru  rA  r‰   rF   rG   rW   1  s   
öüøz)MkldnnRnnLayer.create.<locals>.<listcomp>)rè  r¼  rÛ  r„   rY   r@   r†  rv  rˆ   r   rq   r_   )ré   r‰   rˆ  r‰  rŠ  r‹  rŒ  r�  r  rŽ  r  r�  r�  r‘  r’  r“  r”  rÁ  Z
seq_lengthZ
mini_batchr•  Zhy_shapeZcy_shapeÚresr­  rÅ  r–  Zoutput_sizesZoutput_stridesZ	output_irrF   r—  rG   rè   æ  s\    

÷
ý
ýÿõzMkldnnRnnLayer.create)rF   r‡  )r’   r“   r”   r�   rÍ  rú   r“  r
   rC  rè   rû   rF   rF   rß   rG   r†  Ö  s,      ÿ
ïr†  c                       s\   e Zd Zd
‡ fdd„	Zdd„ Zedeeddddee ee ee eeedœdd	„ƒZ	‡  Z
S )ÚQConvPointWisePT2ErF   c                    s"   t |ƒdk| _tƒ  |||¡ dS )aý  
        if bias is not None
            - inputs = [x, w, b, weight_scale, weight_zp]
            - const_args is: [stride, padding, dilation, groups, x_scale, x_zp, o_inv_scale, o_zp,
              fp32_output, unary_attr, unary_scalars, unary_algorithm]
        else
            - inputs = [x, w, weight_scale, weight_zp]
            - const_args is: [bias, stride, padding, dilation, groups, x_scale, x_zp, o_inv_scale, o_zp,
              fp32_output, unary_attr, unary_scalars, unary_algorithm]
        rO  N©rY   Úhas_biasrÞ   r�   rx  rß   rF   rG   r�   E  s    zQConvPointWisePT2E.__init__c                 C   sh  dd„ | j D ƒ}g }| |  ¡ ¡ |d }|d }| jr@|d n|d }|d |d  }}|dd … \}	}
}}}}}}}}}}d	| _|› d
|› � d
|› � d
|› � d
|› � d
|› � d
|› � d
|	› � d
|
› � d
|› � d
|› � d
|› � d
|› � d
|› � d
|› � d
|› � d
|› � }| |  ¡ › d| j› d|› d�¡ t| jt	ƒ�rd|  
|¡ d S )Nc                 S   s   g | ]}|  ¡ ‘qS rF   rï  r‚  rF   rF   rG   rW   Z  s     z.QConvPointWisePT2E.codegen.<locals>.<listcomp>r   r   rj   éþÿÿÿrò  iôÿÿÿz"torch.ops.onednn.qconv2d_pointwiser  rG  r  r6  ©r­  r·  rî  r›  rÃ  rÉ  rè  r4   rd  r-  rþ  )r�   rÊ  rK   Ú
const_argsr‰   Úpacked_weightrI  Úw_scaleÚw_zpr~   rJ  rK  rL  Úx_scaleÚx_zpÚo_inv_scaleÚo_zpÚfp32_outputrq  rr  rs  rò  rF   rF   rG   rÍ  X  sv    
óÿþýüûúùø	÷
öõôóòñðÿ"zQConvPointWisePT2E.codegenr:   )r‰   r¢  r£  rË  r   r¡  rI  rc  rb  rd  rL  r¤  Úoutput_zero_pointc                 C   s¶   d}d }t | ||||	||
|||ƒ
\}}}}|d krN|d |d  |d< |d< n|d |d  |d< |d< | ¡  | ¡  |||g }|||||||||g }|r¨tj|_t|||d�S ©NFrj   r   r   rf  )rY  rÊ   r7   Úfloat32r   r™  )ré   r‰   r¢  r£  rË  r   r¡  rI  rc  rb  rd  rL  r¤  r§  r¦  rq  rr  rs  rM  rN  r­  rÅ  rX  rv   rF   rF   rG   rè   ‰  sJ    öøýzQConvPointWisePT2E.create)rF   )r’   r“   r”   r�   rÍ  rú   rÿ   rC  r
   rè   rû   rF   rF   rß   rG   r™  D  s$    ü1òr™  c                       sV   e Zd Zd
‡ fdd„	Zdd„ Zeddddee ee ee edddœ
dd	„ƒZ‡  Z	S )ÚQConvPointWiseBinaryPT2ErF   c                    s"   t |ƒdk| _tƒ  |||¡ dS )a~  
        Needs input/weight/output qparams
        if bias is not None
            - inputs = [x, w, b, accum, w_scale, w_zp]
            - const_args = [stride, padding, dilation, groups, x_scale, x_zp, accum_scale, accum_zp, o_inv_scale, o_zp,
            fp32_output, binary_attr, aplha, unary_attr, unary_scalars, unary_algorithm]
        else
            - inputs = [x, w, accum, w_scale, w_zp]
            - const_args = const_args is: [bias, stride, padding, dilation, groups, x_scale, x_zp, accum_scale,
            accum_zp, o_inv_scale, o_zp, fp32_output, binary_attr, aplha, unary_attr, unary_scalars, unary_algorithm]
        é   Nrš  rx  rß   rF   rG   r�   Í  s    z!QConvPointWiseBinaryPT2E.__init__c                 C   s¬  dd„ | j D ƒ}g }| |  ¡ ¡ |d }|d }| jr@|d n|d }|d |d |d   }}}	|d	d … \}
}}}}}}}}}}}}}}}d
| _|› d|› � d|› � d|› � d|› � d|› � d|› � d|› � d|	› � d|› � d|
› � d|› � d|› � d|› � d|› � d|› � d|› � d|› � d|› � d|› � d|› � d|› � }| |  ¡ › d| j› d|› d�¡ t| jt	ƒ�r¨|  
|¡ d S )Nc                 S   s   g | ]}|  ¡ ‘qS rF   rï  r‚  rF   rF   rG   rW   ã  s     z4QConvPointWiseBinaryPT2E.codegen.<locals>.<listcomp>r   r   rj   éýÿÿÿrœ  rò  iðÿÿÿz)torch.ops.onednn.qconv2d_pointwise.binaryr  rG  r  r6  r�  )r�   rÊ  rK   rž  r‰   rŸ  rI  Úaccumr   r¡  r~   rJ  rK  rL  r¢  r£  Úaccum_scaleÚaccum_zpr¤  r¥  r¦  ro  Úalpharq  rr  rs  Z	conv_argsrF   rF   rG   rÍ  á  s’    
ïÿþýüûúùø	÷
öõôóòñðïîíìëÿ"z QConvPointWiseBinaryPT2E.codegenr:   )
r‰   r­  rË  rI  rc  rb  rd  rL  r¤  r§  c                 C   sÔ   d}d }t | |||
||||||ƒ
\}}}}|  ||¡}| |¡ |
d krd|d |d  |d< |d< n|d |d  |d< |d< | ¡  |	 ¡  |||	g }|||||||||||||g }|rÆtj|_t|||d�S r¨  )rY  ré  rp  rÊ   r7   r©  r   rª  )ré   r‰   r¢  r£  r­  r®  r¯  rË  r   r¡  rI  rc  rb  rd  rL  r¤  r§  r¦  ro  r°  rq  rr  rs  rM  rN  r­  rÅ  rX  rW  rF   rF   rG   rè     s`    öû
ôýzQConvPointWiseBinaryPT2E.create)rF   rj  rF   rF   rß   rG   rª  Ì  s    ü9ïrª  c                       sH   e Zd Zd
‡ fdd„	Zdd„ Zedeeddddeedœ	dd	„ƒZ‡  Z	S )ÚQLinearPointwisePT2ErF   c                    s"   t |ƒdk| _tƒ  |||¡ dS )a·  
        if bias is not None
            - inputs = [x, w, b, weight_scale, weight_zp]
            - const_args is: [x_scale, x_zp, o_inv_scale, o_zp,
              fp32_output, unary_attr, unary_scalars, unary_algorithm]
        else
            - inputs = [x, w, weight_scale, weight_zp]
            - const_args is: [bias, x_scale, x_zp, o_inv_scale, o_zp,
              fp32_output, unary_attr, unary_scalars, unary_algorithm]
        rO  Nrš  rx  rß   rF   rG   r�   o  s    zQLinearPointwisePT2E.__init__c                 C   s8  dd„ | j D ƒ}g }| |  ¡ ¡ |d }|d }| jr@|d n|d }|d |d  }}|dd … \}	}
}}}}}}d	| _|› d
|	› � d
|
› � d
|› � d
|› � d
|› � d
|› � d
|› � d
|› � d
|› � d
|› � d
|› � d
|› � }| |  ¡ › d| j› d|› d�¡ t| jt	ƒ�r4|  
|¡ d S )Nc                 S   s   g | ]}|  ¡ ‘qS rF   rï  r‚  rF   rF   rG   rW   „  s     z0QLinearPointwisePT2E.codegen.<locals>.<listcomp>r   r   rj   rœ  rò  iøÿÿÿz"torch.ops.onednn.qlinear_pointwiser  rG  r  r6  r�  )r�   rÊ  rK   rž  r‰   rŸ  rI  r   r¡  r¢  r£  r¤  r¥  r¦  rq  rr  rs  rò  rF   rF   rG   rÍ  ‚  s^    

÷ÿþýüûúùø	÷
öõôÿ"zQLinearPointwisePT2E.codegenr:   )	r‰   r¢  r£  rË  r   r¡  rI  r¤  r§  c              	   C   sd   t | |||ƒ\}}}}| ¡  | ¡  |||g }|||||	|
|||g }|
rVtj|_t|||d�S re  )r[  rÊ   r7   r©  r   r±  )ré   r‰   r¢  r£  rË  r   r¡  rI  r¤  r§  r¦  rq  rr  rs  r­  rÅ  rX  rv   rF   rF   rG   rè   «  s4    üøýzQLinearPointwisePT2E.create)rF   )
r’   r“   r”   r�   rÍ  rú   rÿ   rC  rè   rû   rF   rF   rß   rG   r±  n  s    ü)ör±  c                   @   sB   e Zd ZU dZeed< dd„ Zdd„ Zedd„ ƒZ	d	d
„ Z
e
ZdS )rm  zC
    TensorBox / StorageBox allow in-place mutation of Tensors
    re  c                 C   s8   t | j|ƒ}t|ƒr|S tt| jƒj› d|› d�ƒ‚d S )Nr)  z not callable)rJ   re  ÚcallableÚAttributeErrorrA   r’   )r�   rN   rO   rF   rF   rG   Ú__getattr__å  s    zMutableBox.__getattr__c                 C   s
   | j  ¡ S rI   rí  rŽ   rF   rF   rG   rÊ   ë  s    zMutableBox.realizec                 C   s   | j jS rI   rä  rŽ   rF   rF   rG   rd  î  s    zMutableBox.layoutc                 C   sn   t | jtƒr8t| ƒj› dt| jƒj› d�}d}| jj}nt| ƒj› d�}| j}d}|tt|ƒƒ|g}d |¡S )Nr  z))r6  Ú
)r4   re  rm  rA   r’   r·   rP   r¸   )r�   Zline0Zendlrä  r»   rF   rF   rG   rÜ   ò  s    

ýzMutableBox.__str__N)r’   r“   r”   r:  r¨   rË   r´  rÊ   r[  rd  rÜ   r‘   rF   rF   rF   rG   rm  Ý  s   

rm  c                   @   s   e Zd Zedd„ ƒZdS )r:   c                 C   s   t t| ƒƒS rI   )r:   rÙ  )re  rF   rF   rG   rè     s    zTensorBox.createN)r’   r“   r”   rÌ   rè   rF   rF   rF   rG   r:     s   c                   @   sL   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zedd„ ƒZ	edd„ ƒZ
dS )rÙ  c                 C   s&   t | jttfƒr"| j ¡ tjjkS dS rå   )r4   re  r|  rÝ  rè  r1   r�   Zgraph_inputsrŽ   rF   rF   rG   Úis_input_buffer  s    zStorageBox.is_input_bufferc                 C   s´   t | jtttttfƒr | j ¡ S t | jtt	fƒs>t
t| jƒƒ‚| j ¡ }| j ¡ }td t| j ¡ | j ¡ | j ¡ d�| jd�| _tj | j¡| j_| j| j_|| j_|| j_| jjS )Nrb  rc  )r4   re  rk  r«  r|  rÝ  rª  rè  r  r9  r@   rA   rä   r²   rõ   rˆ   r‡   r„   r1   r�   r¯  rN   rª   rÚ   r¯   )r�   rÚ   r¯   rF   rF   rG   rÊ     s6    ûþ



ýù	
zStorageBox.realizec                 C   s0   t | jttfƒr,|  ¡ dkr,|  ¡ r,|  ¡  dS )zL
        Called on buffers we expect to be forced to realize later.
        r   N)r4   re  r  r9  rƒ  Ú8is_pointwise_non_scalar_tensor_num_reads_larger_than_onerÊ   rŽ   rF   rF   rG   rî  0  s    ÿ
þýzStorageBox.realize_hintc                 C   s(   t | jtƒo&|  ¡ tjkp&|  ¡ tjkS rI   )r4   re  r  rƒ  r   Zrealize_acc_reads_thresholdrï   Úrealize_bytes_thresholdrŽ   rF   rF   rG   rì  ;  s    þz!StorageBox.has_exceeded_max_readsc                 C   sn   t ttf dœdd„}|dkrjt| jttfƒrj|  ¡ tjksbt|  	¡ ƒtj
ksbt| jƒrj|| jƒrj|  ¡  dS )zj
        A heuristic to decide if we should realize a tensor
        that is used multiple times.
        )Úloopsc                    s$   dg}|   ¡ ‰ t‡ fdd„|D ƒƒS )zW
            The heuristic for realizing reused result of heavy ops on cpu
            Úexpc                 3   s   | ]}|d  ˆ kV  qdS )r  NrF   )rS   Úop©Zfn_strrF   rG   rj  M  s     zGStorageBox.mark_reuse.<locals>.should_realize_on_cpu.<locals>.<genexpr>)rÙ   r  )r¹  Z	heavy_opsrF   r¼  rG   Úshould_realize_on_cpuG  s    z4StorageBox.mark_reuse.<locals>.should_realize_on_cpur   N)r   r  r9  r4   re  rƒ  r   Zrealize_reads_thresholdrY   rÙ   r¸  r§   rÊ   )r�   rë  r½  rF   rF   rG   ré  A  s    	ÿþüûúú	zStorageBox.mark_reusec                 C   sz   | j }t|tttfƒrdS t|tƒr.| ¡ }nBt|ttfƒsHt	t
|ƒƒ‚td t| ¡ | ¡ | ¡ d�|d� ¡ }t|jƒS )Nr   rb  rc  )re  r4   r«  r|  rÝ  rk  rl  r  r9  r@   rA   rõ   rˆ   r‡   r„   rY   rù   )r�   re  rt  rF   rF   rG   rƒ  Z  s     

ýù
	zStorageBox.num_readsc                 C   s,   t | jtƒr(tdd„ | j ¡ D ƒƒdkS dS )Nc                 s   s   | ]}|j d kV  qdS )r   Nr[   )rS   ÚreadrF   rF   rG   rj  r  s     zVStorageBox.is_pointwise_non_scalar_tensor_num_reads_larger_than_one.<locals>.<genexpr>r   T)r4   re  r  r#  rÂ   rŽ   rF   rF   rG   r·  n  s    
ÿýzCStorageBox.is_pointwise_non_scalar_tensor_num_reads_larger_than_oneN)r’   r“   r”   r¶  rÊ   rî  rì  ré  r%   rƒ  r·  rF   rF   rF   rG   rÙ    s   
rÙ  c                       sV   e Zd Zee d¡dd„ ƒƒZ‡ fdd„Zej	j
edœ‡ fdd„Z‡ fd	d
„Z‡  ZS )ÚInterpreterShimNc                   C   s   t j t¡S rI   )r7   rÏ   Zsymbolic_tracer   rF   rF   rF   rG   Ú	_dummy_gmy  s    zInterpreterShim._dummy_gmc                    s>   t ƒ j|  ¡ dd� | | _|| _|| _d| _|j| _d | _	d S )NF)Zgarbage_collect_values)
rÞ   r�   rÀ  Úmoduler�   Ú
submodulesZextra_tracebackr`  Z
fetch_attrÚcurrent_node©r�   r�   rÂ  rß   rF   rG   r�   ~  s    zInterpreterShim.__init__)rì   r«  c                    s   || _ tƒ  |¡S rI   )rÃ  rÞ   Úrun_node)r�   rì   rß   rF   rG   rÅ  ‰  s    zInterpreterShim.run_nodec              
      s0   t  | ¡� tƒ j||ŽW  5 Q R £ S Q R X d S rI   )r1   Zset_interpreter_handlerrÞ   Úrun)r�   rK   rL   rß   rF   rG   rÆ  �  s    zInterpreterShim.run)r’   r“   r”   rÌ   r„  Ú	lru_cacherÀ  r�   r7   rÏ   rÐ   r   rÅ  rÆ  rû   rF   rF   rß   rG   r¿  x  s   r¿  c                       sx   e Zd ZdZ‡ fdd„Zedd„ ƒZedd„ ƒZdd	„ Ze	j
d
œdd„Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Z‡  ZS )rž  z˜
    Captures the body of a Loops subclass into an FX graph.  Persists any
    indexing simplifications and makes it easier to analyze loop bodies.
    c                    sj   t ƒ  ¡  || _i | _i | _g | _g | _i | _i | _g | _	d| j
i| _i | _g | _t| ||ƒ| _d | _d S )NÚ	get_index)rÞ   r�   r£  rŸ  Úindexing_exprs_namerù   Úwritesr   r¡  rX  rÈ  rÂ  Ú	subblocksÚindirect_varsÚLoopBodyBlockÚ
root_blockÚindexing)r�   rO   rK   r£  rß   rF   rG   r�   ˜  s    
zLoopBody.__init__c                 C   s0   t  | jjfdd„ | j ¡ D ƒ¡}dd„ |D ƒS )Nc                 s   s   | ]}|j V  qd S rI   )r�   )rS   ÚblockrF   rF   rG   rj  ¬  s     z%LoopBody.get_nodes.<locals>.<genexpr>c                 S   s   g | ]}|j D ]}|‘qqS rF   )rB   )rS   r�   rC   rF   rF   rG   rW   ®  s       z&LoopBody.get_nodes.<locals>.<listcomp>)r†  ÚchainrÎ  r�   rË  rÅ  )r�   Z
all_graphsrF   rF   rG   Ú	get_nodes¨  s
    þzLoopBody.get_nodesc                 C   s   ddl m} || ƒS )Nr   )Ú	BoundVars)ÚboundsrÓ  )r�   rÓ  rF   rF   rG   rÔ  °  s    zLoopBody.boundsc                 C   s`   dt | jƒ› �g}| dd„ | j ¡ D ƒ¡ | dd„ t d| jfg| j ¡ ¡D ƒ¡ d 	|¡S )Nzvar_ranges = c                 S   s   g | ]\}}|› d |› �‘qS )rG  rF   )rS   rN   r•  rF   rF   rG   rW   ¹  s     z&LoopBody.debug_str.<locals>.<listcomp>c                 S   s   g | ]\}}|  |¡‘qS rF   )Ú	debug_str)rS   rN   rÐ  rF   rF   rG   rW   »  s   ÿr¯  rµ  )
r^   r£  r·  rŸ  r¢  r†  rÑ  rÎ  rË  r¸   rº   rF   rF   rG   rÕ  ·  s    
 ÿþÿzLoopBody.debug_str)r‚  c                 C   sd   t | |ƒ |¡ |d k	r,|t | |› d�ƒ|< || jkrZdt| jƒ› �}|| j|< || j|< | j| S )NZ
_name2exprrU   )rJ   rp  rÉ  rY   rŸ  )r�   r‚  ÚcategoryÚbuf_namerN   rF   rF   rG   Úadd_index_exprÄ  s    


zLoopBody.add_index_exprc                 C   s<   |d   ¡ r|| jkr|}n|› t| jƒ› �}|| j|< |S )zaNot actually for nn.Modules, but subblocks in generated code are mapped to FX call_module opcodesrò  )Ú	isnumericrÂ  rY   )r�   rÐ  r3   rN   rF   rF   rG   Úadd_submoduleÎ  s
    
zLoopBody.add_submodulec                 C   s(   dt | jƒ› �}t|ƒ}| j |¡ |S )NZindirect)rY   rÌ  r.   rp  )r�   r}   rN   r+  rF   rF   rG   Úadd_indirect×  s    zLoopBody.add_indirectc                    sB   t ˆƒt ˆ ƒkrdS | jdk	s"t‚‡ ‡fdd„| j ¡ D ƒ| _dS )z,Swap in a variable used in indirect indexingNc                    s    i | ]\}}|t |ˆˆ iƒ“qS rF   r&  rú  ©Únewr«   rF   rG   rn   â  s      z-LoopBody.replace_indirect.<locals>.<dictcomp>)rP   rÏ  r@   r¢  )r�   r«   rÝ  rF   rÜ  rG   Úreplace_indirectÝ  s    zLoopBody.replace_indirectc                 C   s   | j d k	st‚| j | S rI   )rÏ  r@   r¾   rF   rF   rG   rÈ  ä  s    zLoopBody.get_indexc                    sˆ   t tj|Ž ƒ}t|ƒtˆjƒks.t|ˆjfƒ‚t‡fdd„|D ƒƒsHt‚ttˆj 	¡ |ƒƒ‰ ‡ fdd„ˆj
 ¡ D ƒˆ_ˆ ¡ }d ˆ_|S )Nc                 3   s   | ]}|ˆ j kV  qd S rI   )r£  rˆ  rŽ   rF   rG   rj  ë  s     z$LoopBody.__call__.<locals>.<genexpr>c                    s   i | ]\}}|t |ˆ ƒ“qS rF   r&  )rS   rN   r‚  r'  rF   rG   rn   í  s   ÿ z%LoopBody.__call__.<locals>.<dictcomp>)r5   r†  rÑ  rY   r£  r@   ro  r^   r_   r|  rŸ  r¢  rÏ  rÎ  )r�   ru  rU   r\  rF   )r(  r�   rG   Ú__call__è  s     
þzLoopBody.__call__)r’   r“   r”   r:  r�   r%   rÒ  rÔ  rÕ  r;   r   rØ  rÚ  rÛ  rÞ  rÈ  rß  rû   rF   rF   rß   rG   rž  ’  s   


	rž  c                   @   s@   e Zd ZdZeedef ee dœdd„Zdd„ Z	dd	d
„Z
dS )rÍ  a  
    Captures the body of a Loops subclass into an FX graph.
    In normal cases there will be a 1:1 mapping between LoopBody and
    LoopBodyBlock, hower in the case of ops.masked() the masked out
    operations will manifest as an extra LoopBodyBlock.
    .)r¯  rO   rK   c           	   	      sÄ   |ˆ_ d‡‡fdd„	‰ G ‡ ‡‡fdd„dtjƒ}tj ¡ ‰tjjˆjd�ˆ_ˆ 	dddi ¡}d	d
l
m} d	dlm} |||ƒˆj jƒ}tjr”||ƒ}t |¡� t ||Ž ¡ W 5 Q R X ˆjˆ_d S )Nc              	      s   ˆ  ddˆ j | ||¡fi ¡S )NÚcall_modulerÈ  )Úcreate_proxyr¯  rØ  )r‚  rÖ  r×  ©r�   ÚtracerrF   rG   Ú	add_index  s    üz)LoopBodyBlock.__init__.<locals>.add_indexc                       s¶   e Zd Zd”_eejdœ‡ fdd„Zd‡ fdd„	Z‡ fdd	„Z	d
d„ Z
‡ fdd„Zeejejedœ‡ fdd„Zeedef dœ‡‡fdd„ƒZed‡‡fdd„	ƒZe‡fdd„ƒZdS )z/LoopBodyBlock.__init__.<locals>.CaptureIndexingÚCaptureIndexing)rN   rU   c                    s   ˆ |d|ƒ}| j  ||¡S )Nrù   )Ú_innerr2  )r�   rN   rU   ©rä  rF   rG   r2    s    z4LoopBodyBlock.__init__.<locals>.CaptureIndexing.loadNc                    s   ˆ |d|ƒ}| j  ||||¡S ©NrÊ  )ræ  r  )r�   rN   rU   rŸ   r  rç  rF   rG   r    s    z5LoopBodyBlock.__init__.<locals>.CaptureIndexing.storec                    s   ˆ |d|ƒ}| j  |||¡S rè  )ræ  r?  )r�   rN   rU   rŸ   rç  rF   rG   r?    s    z?LoopBodyBlock.__init__.<locals>.CaptureIndexing.store_reductionc                    s8   | j  ||||¡‰ d|kr4t‡ fdd„tdƒD ƒƒS ˆ S )Nr¼  c                 3   s   | ]}ˆ | V  qd S rI   rF   rR   ©r\  rF   rG   rj    s     zLLoopBodyBlock.__init__.<locals>.CaptureIndexing.reduction.<locals>.<genexpr>ri   )ræ  r>  r6   rZ   )r�   r   r;  r3  rŸ   rF   ré  rG   r>    s    z9LoopBodyBlock.__init__.<locals>.CaptureIndexing.reductionc                    s:   t |ttjfƒr"| j t|ƒ|¡S ˆ |dƒ}| j ||¡S ©NrX  )r4   rC  r;   r   ræ  rþ   rŠ  )r�   rU   r   rç  rF   rG   rŠ    s    
z:LoopBodyBlock.__init__.<locals>.CaptureIndexing.index_expr)Úoffsets_nameÚoffsets_sizeÚindexing_dtyperN  c                    s   ˆ |dƒ}| j  |||||¡S rê  )ræ  Ú	bucketize)r�   rÅ  rë  rì  rí  rN  rç  rF   rG   rî  $  s    
    ÿz9LoopBodyBlock.__init__.<locals>.CaptureIndexing.bucketize.)Úmasked_bodyc                    sH   ‡ fdd„}ˆj  |d¡}tˆj |g ƒ‰ ˆ ˆj j|< ˆ d|| |fi ¡S )zb
                Recursively capture the masked out body in another LoopBodyBlock
                c                    s   t j | ˆ |¡S rI   )r1   r0   r°  )r0  rX  ©ZsubblockrF   rG   Úshim7  s    zDLoopBodyBlock.__init__.<locals>.CaptureIndexing.masked.<locals>.shimZmasked_subblockrà  )r¯  rÚ  rÍ  rË  rá  )Z
mask_proxyrï  Zother_proxyrñ  rN   râ  rð  rG   r°  1  s       ÿz6LoopBodyBlock.__init__.<locals>.CaptureIndexing.maskedTc                    sD   ‡ ‡‡‡fdd„}ˆj  ˆ¡‰ˆ dˆj  |dˆ› �¡| fi ¡ ˆS )z�
                Flow data from tensors into indexing formulas.
                Introduce a call_module to update the indexing.
                c                    s   ˆj  ˆtj | ˆˆ ¡¡ d S rI   )r¯  rÞ  r1   r0   Úindirect_indexing)Znew_var)Úcheckr�   r}   r+  rF   rG   Úset_indirectH  s     ÿzWLoopBodyBlock.__init__.<locals>.CaptureIndexing.indirect_indexing.<locals>.set_indirectrà  Úset_)r¯  rÛ  rá  rÚ  )Zindex_proxyr}   ró  rô  râ  )ró  r}   r+  rG   rò  A  s    üzALoopBodyBlock.__init__.<locals>.CaptureIndexing.indirect_indexingc                    s   ˆ   dd| fi ¡ d S )Nr>  )rá  ré  )rã  rF   rG   r>  V  s    z6LoopBodyBlock.__init__.<locals>.CaptureIndexing.output)N)T)r’   r“   r”   rN   rP   r;   r   r2  r  r?  r>  rŠ  r7   r   r“  rî  rÌ   r   r   r°  rò  r>  rF   ©rä  r�   rã  rF   rG   rå  	  s"   	úrå  )Z
tracer_clsÚplaceholderr0   rF   r   )ÚIndexPropagation)ÚSimplifyIndexing)N)r¯  r1   ZWrapperHandlerr7   rÏ   ZTracerZGraphrà   r�   rá  Zindex_propagationrø  r‚   rù  r£  r   Zconstant_and_index_propagationZset_ops_handlerr0   r>  )	r�   r¯  rO   rK   rå  Z	proxy_opsrø  rù  ÚhandlerrF   rö  rG   r�   þ  s    Q
zLoopBodyBlock.__init__c                 C   s"   | j }| jj}t||ƒ t ¡ ¡S rI   )r�   r¯  rÂ  r¿  rÆ  r1   Zget_ops_handlerrÄ  rF   rF   rG   rß  k  s    zLoopBodyBlock.__call__rÐ  c              
   C   s8   t j | jj| j¡j}t dd| 	¡  
dd|› d�¡¡S )Nz;[^\n]*r³   zdef forward(zdef r  )r7   rÏ   ZGraphModuler¯  rÂ  r�   ÚcodeÚreÚsubÚstripr.  )r�   rN   rû  rF   rF   rG   rÕ  q  s    üzLoopBodyBlock.debug_strN)rÐ  )r’   r“   r”   r:  rž  r   r   r
   r�   rß  rÕ  rF   rF   rF   rG   rÍ  ö  s   mrÍ  c                       sL   e Zd ZdZd‡ fdd„	Zdd„ Zdd„ Zed	d
œdd„ƒZdd„ Z	‡  Z
S )ÚWaitz‘
    Wait should not be used by itself.  It should always be constructed in tandem
    with a collective op that produces a work to wait on.
    rF   c                    s   t ƒ  |||¡ d S rI   ©rÞ   r�   rx  rß   rF   rG   r�   �  s    zWait.__init__c                 C   s   dS rå   rF   rŽ   rF   rF   rG   r©  ‰  s    zWait.should_allocatec                 C   sN   |  d¡ dd„ | jD ƒ\}| |› d|› d�¡ | |  ¡ › d|› �¡ d S )NzGfrom torch.distributed._functional_collectives_impl import _wait_tensorc                 s   s   | ]}|  ¡ V  qd S rI   rï  r  rF   rF   rG   rj  �  s     zWait.codegen.<locals>.<genexpr>z = _wait_tensor(r6  rG  )Úadd_import_oncer­  rÉ  rè  )r�   rÊ  Zinput_collectiverF   rF   rG   rÍ  Œ  s    ÿzWait.codegenr:   )Úcollective_opc                 C   s   |  ¡  t| ¡ |gd�S )N)rd  r­  )rr  rÿ  r†   )ré   r  rF   rF   rG   rè   ™  s
    þzWait.createc                 C   s   | j d  ¡ gS rÄ   )r­  r9  rŽ   rF   rF   rG   rz  ¢  s    zWait.get_alias_names)rF   )r’   r“   r”   r:  r�   r©  rÍ  rú   rè   rz  rû   rF   rF   rß   rG   rÿ  {  s   	 ürÿ  c                       sD   e Zd ZdZ‡ fdd„Zdd„ Zdd„ Zedd	„ ƒZd
d„ Z	‡  Z
S )ÚCollectiveKernela  
    Each collective should follow the pattern:
    - extend InPlaceCollectiveKernel or OutOfPlaceCollectiveKernel.
    - the kernel delegates into c10d processgroup, which returns a 'work' obj
    - the work obj is registered via _register_tensor_work so it can be waited on later
    c                    s$   t ƒ  d |||¡ tj | ¡| _d S rI   ©rÞ   r�   r1   r�   r¯  rN   rx  rß   rF   rG   r�   ¯  s    zCollectiveKernel.__init__c                 C   s   t dƒ‚d S ©NzMust implementrá  ©r�   rÊ  r  Úinput_namesrF   rF   rG   Úcodegen_collective³  s    z#CollectiveKernel.codegen_collectivec                 C   s   t dƒ‚d S r  rá  r  rF   rF   rG   Úcodegen_output·  s    zCollectiveKernel.codegen_outputc                 C   s   dd„ }t t||ƒƒS )Nc                 S   s(   t t|  ¡ |  ¡ |  ¡ ƒ| ƒ}t |¡S rI   )ÚInPlaceHintrõ   rˆ   r‡   r„   r:   rè   )r+  r»  rF   rF   rG   Ú
wrap_input½  s
     ÿz;CollectiveKernel.wrap_inputs_as_inplace.<locals>.wrap_input)r5   r¹   )ré   r­  r  rF   rF   rG   Úwrap_inputs_as_inplace»  s    z'CollectiveKernel.wrap_inputs_as_inplacec              
   C   sœ   |  d¡ |  d¡ |  d¡ dd„ | jD ƒ}|  ¡ }| j\}}}| |› d|› d|› d|› d	�¡ |  |||¡ |  |||¡ | d
|› d|› d�¡ d S )Nz import torch.distributed as distz1import torch.distributed.distributed_c10d as c10dzEimport torch.distributed._functional_collectives_impl as fun_col_implc                 S   s   g | ]}|  ¡ ‘qS rF   rï  r  rF   rF   rG   rW   Ì  s     z,CollectiveKernel.codegen.<locals>.<listcomp>z0_pg = c10d._find_or_create_pg_by_ranks_and_tag('rJ  r  r6  z#fun_col_impl._register_tensor_work(z_work))r  r­  rè  rÅ  rÉ  r	  r  )r�   rÊ  r  r  ÚtagÚranksÚ
group_sizerF   rF   rG   rÍ  Å  s     

ÿÿÿzCollectiveKernel.codegen)r’   r“   r”   r:  r�   r  r	  rú   r  rÍ  rû   rF   rF   rß   rG   r  §  s   
	r  c                       s8   e Zd ZdZ‡ fdd„Zdd„ Zdd„ Zdd	„ Z‡  ZS )
ÚInPlaceCollectiveKernelz¤
    InPlaceCollectiveKernel are those with in-out arguments such as all_reduce.
    Extend this kernel if your collective needs to modify its inputs in-place.
    c                    s   t ƒ  |||¡ d S rI   r   rx  rß   rF   rG   r�   â  s    z InPlaceCollectiveKernel.__init__c                 C   s   dS rå   rF   rŽ   rF   rF   rG   r©  å  s    z'InPlaceCollectiveKernel.should_allocatec                 C   s   dS r¨  rF   rŽ   rF   rF   rG   Úhas_side_effectsè  s    z(InPlaceCollectiveKernel.has_side_effectsc                 C   sF   t |ƒdkr*| |› dd |¡› d�¡ n| |› d|d › �¡ d S )Nr   ú = [ú,z] rG  r   )rY   rÉ  r¸   r  rF   rF   rG   r	  ë  s    z&InPlaceCollectiveKernel.codegen_output)	r’   r“   r”   r:  r�   r©  r  r	  rû   rF   rF   rß   rG   r  Ü  s
   r  c                       sR   e Zd ZdZ‡ fdd„Zdd„ Zdd„ Zdd	„ Zeddd„ƒZ	edd„ ƒZ
‡  ZS )ÚOutOfPlaceCollectiveKernelz‚
    OutOfPlaceCollectiveKernel are those that allocate their
    outputs and leave their inputs inplace, such as all_gather.
    c                    s$   t ƒ  ||| |¡ || _|| _d S rI   )rÞ   r�   ÚoutputsÚoriginal_inputs©r�   rd  r­  r  rÅ  rß   rF   rG   r�   ø  s    z#OutOfPlaceCollectiveKernel.__init__c                 C   s   dS rå   rF   rŽ   rF   rF   rG   r©  ý  s    z*OutOfPlaceCollectiveKernel.should_allocatec                 C   s   dS r¨  rF   rŽ   rF   rF   rG   r     s    z+OutOfPlaceCollectiveKernel.has_side_effectsc                 C   sX   dd„ | j D ƒ}| |› dd |¡› d�¡ | |› dd dd„ | jD ƒ¡› d�¡ d S )	Nc                 S   s   g | ]}|  ¡ ‘qS rF   rï  r  rF   rF   rG   rW     s     z=OutOfPlaceCollectiveKernel.codegen_output.<locals>.<listcomp>z_inputs = [r  rñ  r  c                 s   s   | ]}|j V  qd S rI   rM   r‚  rF   rF   rG   rj    s     z<OutOfPlaceCollectiveKernel.codegen_output.<locals>.<genexpr>)r  rÉ  r¸   r  r  rF   rF   rG   r	    s    z)OutOfPlaceCollectiveKernel.codegen_outputNc                 C   sP   g }|D ]B}|  ¡ }|d k	r$||ƒ tt| ¡ | ¡ |d�d�}| |¡ q|S )Nrb  r5  )r„   ÚOutputBufferrõ   rˆ   r‡   rp  )ré   r­  Zsize_cbr  rº  ró  ZbuffrF   rF   rG   Úcreate_output_buffers  s    ýÿz0OutOfPlaceCollectiveKernel.create_output_buffersc                    s   ‡ fdd„t |ƒD ƒS )Nc                    s&   g | ]\}}t |jˆ d |› d�ƒ‘qS )rð  rñ  )ÚMultiOutputNoSizeAssertrd  )rS   rT   Zout_t©ÚcollrF   rG   rW     s   û
ýzBOutOfPlaceCollectiveKernel.create_output_nodes.<locals>.<listcomp>rí   )ré   r  Zoutput_buffersrF   r  rG   Úcreate_output_nodes  s    
úz.OutOfPlaceCollectiveKernel.create_output_nodes)N)r’   r“   r”   r:  r�   r©  r  r	  rú   r  r  rû   rF   rF   rß   rG   r  ò  s   r  c                       s0   e Zd ZdZdd„ Z‡ fdd„Zdd„ Z‡  ZS )r
  añ  
    Helper OP to encode an in/out argument that tries to make it inplace whenever possible.
    Wrap the input of your inplace op to enable this behavior.

    The design is based on two key decisions:
    - this node is resposible for allocating the in/out buffer used by the collective.
        This is controlled by the ``should_allocate`` method that returns True here and
        False for the collective node
    - The scheduler special-case this node and enable it to reuse its input.
    c                 C   sB   | j d  ¡ }|  ¡ }| | | j d ¡s>| |› d|› d�¡ d S )Nr   r"  z) #no reuse)r­  r9  rè  Z	did_reuserÉ  )r�   rÊ  Z
input_namer  rF   rF   rG   rÍ  3  s    zInPlaceHint.codegenc                    s6   |   |¡}tƒ  d ||  |g¡d¡ tj | ¡| _d S rB  )r¼  rÞ   r�   r¬  r1   r�   r¯  rN   )r�   rd  rº  rß   rF   rG   r�   9  s    
zInPlaceHint.__init__c                 C   s   dS r¨  rF   rŽ   rF   rF   rG   r©  >  s    zInPlaceHint.should_allocate)r’   r“   r”   r:  rÍ  r�   r©  rû   rF   rF   rß   rG   r
  '  s   r
  c                       s0   e Zd ZdZ‡ fdd„Zdd„ Zdd„ Z‡  ZS )r  zO
    Represent the output buffer used by ops that require multiple of them
    c                    s$   t ƒ jd |g d� tj | ¡| _d S )NrÀ  r  )r�   rd  rß   rF   rG   r�   G  s    zOutputBuffer.__init__c                 C   s   dS r¨  rF   rŽ   rF   rF   rG   r©  K  s    zOutputBuffer.should_allocatec                 C   s   |  d| j› �¡ d S )Nz# collective out buffer )rÉ  rN   rÌ  rF   rF   rG   rÍ  N  s    zOutputBuffer.codegen)r’   r“   r”   r:  r�   r©  rÍ  rû   rF   rF   rß   rG   r  B  s   r  c                       s(   e Zd ZdZ‡ fdd„Zdd„ Z‡  ZS )r  z§
    Extract partial output from a multi-output OP.
    Works like MultiOutput but doesn't assert size. This must be a property guaranteed by the op emiting this.
    c                    s   t ƒ  ||g ¡ || _d S rI   )rÞ   r�   rU   )r�   rd  rº  rU   rß   rF   rG   r�   X  s    z MultiOutputNoSizeAssert.__init__c                 C   s,   |  |  ¡ › d| jd  ¡ › | j› �¡ d S rF  )rÉ  rè  r­  rU   rÌ  rF   rF   rG   rÍ  \  s     ÿzMultiOutputNoSizeAssert.codegen)r’   r“   r”   r:  r�   rÍ  rû   rF   rF   rß   rG   r  R  s   r  c                       sN   e Zd Z‡ fdd„Zdd„ Zeed eeee edœdd„ƒZ	d	d
„ Z
‡  ZS )ÚAllReduceCoalescedc                    s   t ƒ  |||¡ || _d S rI   ©rÞ   r�   Ú	reduce_op©r�   rd  r­  rÅ  r   rß   rF   rG   r�   c  s    zAllReduceCoalesced.__init__c                 C   s   dS rå   rF   rŽ   rF   rF   rG   r©  g  s    z"AllReduceCoalesced.should_allocater:   ©r­  r   r  r  r  c           	      C   s4   |   |¡}t|d  ¡ ƒ}t|||||g|d�}|S ©Nr   )rd  r­  rÅ  r   )r  rv  rˆ   r  )	ré   r­  r   r  r  r  Úinplace_inputsrd  rv   rF   rF   rG   rè   j  s    	
üzAllReduceCoalesced.createc              
   C   s,   |  |› d|› dt| jƒ› d|› d�¡ d S )Nz"_work = dist.all_reduce_coalesced(z%, op=fun_col_impl._str_to_reduce_op('ú
'), group=ú_pg, async_op=True)©rÉ  rP   r   r  rF   rF   rG   r  ~  s     ÿz%AllReduceCoalesced.codegen_collective)r’   r“   r”   r�   r©  rú   r
   rP   rC  rè   r  rû   rF   rF   rß   rG   r  b  s   úr  c                       sB   e Zd Z‡ fdd„Zedeeee edœdd„ƒZdd„ Z	‡  Z
S )	Ú	AllReducec                    s   t ƒ  |||¡ || _d S rI   r  r!  rß   rF   rG   r�   ‰  s    zAllReduce.__init__r:   ©r‰   r   r  r  r  c           	      C   s:   |   |g¡}t|d  ¡ ƒ}t|||||g|d�}|d S r#  )r  rv  rˆ   r(  )	ré   r‰   r   r  r  r  r$  rd  rv   rF   rF   rG   rè   �  s    üzAllReduce.createc              
   C   s,   |  |› d|› d|› dt| jƒ› d�¡ d S )Nz_work = dist.all_reduce(z, async_op=True, group=ú(_pg, op=fun_col_impl._str_to_reduce_op('ú'))r'  r  rF   rF   rG   r  œ  s     ÿzAllReduce.codegen_collective©r’   r“   r”   r�   rú   rP   r
   rC  rè   r  rû   rF   rF   rß   rG   r(  ˆ  s       ÿr(  c                       s@   e Zd Z‡ fdd„Zedeee edœdd„ƒZdd„ Z	‡  Z
S )	ÚAllGatherIntoTensorc                    s   t ƒ  ||||¡ d S rI   r   r  rß   rF   rG   r�   ¤  s    zAllGatherIntoTensor.__init__r:   )r‰   r  r  r  c           
         sZ   |   |¡g}‡ fdd„}|  ||¡}t|d  ¡ ƒ}t|||||ˆ gd�}	|  |	|¡d S )Nc                    s   | d  ˆ 9  < d S rÄ   rF   ©ró  ©r  rF   rG   Úcompute_size«  s    z0AllGatherIntoTensor.create.<locals>.compute_sizer   ©rd  r­  r  rÅ  )r¼  r  rv  rˆ   r-  r  )
ré   r‰   r  r  r  r­  r0  r  rd  rA  rF   r/  rG   rè   §  s    üzAllGatherIntoTensor.createc              
   C   s&   |  |› d|› d|› d|› d�¡ d S )Nz$_work = dist.all_gather_into_tensor(ú[0], ú!_inputs[0], async_op=True, group=z_pg)©rÉ  r  rF   rF   rG   r  º  s    ÿz&AllGatherIntoTensor.codegen_collectiver,  rF   rF   rß   rG   r-  £  s   r-  c                       sB   e Zd Z‡ fdd„Zedeeee edœdd„ƒZdd„ Z	‡  Z
S )	ÚReduceScatterTensorc                    s   t ƒ  ||||¡ || _d S rI   r  ©r�   rd  r­  r  rÅ  r   rß   rF   rG   r�   Â  s    zReduceScatterTensor.__init__r:   r)  c                    s\   |   |¡g}‡ fdd„}|  ||¡}t|d  ¡ ƒ}	t|	||||ˆ g|d�}
|  |
|¡d S )Nc                    s   | d  ˆ   < d S rÄ   rF   r.  r/  rF   rG   r0  Ñ  s    z0ReduceScatterTensor.create.<locals>.compute_sizer   ©rd  r­  r  rÅ  r   )r¼  r  rv  rˆ   r5  r  )ré   r‰   r   r  r  r  r­  r0  r  rd  rA  rF   r/  rG   rè   Æ  s    	ûzReduceScatterTensor.createc                 C   s2   |  |› d|› d|› d|› dt| jƒ› d�
¡ d S )Nz#_work = dist.reduce_scatter_tensor(r2  r3  r*  r+  r'  r  rF   rF   rG   r  á  s    &ÿz&ReduceScatterTensor.codegen_collectiver,  rF   rF   rß   rG   r5  Á  s   úr5  c                       sD   e Zd Z‡ fdd„Zeed eee edœdd„ƒZdd„ Z	‡  Z
S )	ÚAllGatherIntoTensorCoalescedc                    s   t ƒ  ||||¡ d S rI   r   r  rß   rF   rG   r�   ê  s    z%AllGatherIntoTensorCoalesced.__init__r:   )r­  r  r  r  c           	         sT   ‡ fdd„|D ƒ}‡fdd„}ˆ   ||¡}t|d  ¡ ƒ}t|||||ˆgd�}|S )Nc                    s   g | ]}ˆ   |¡‘qS rF   rÖ  r‚  r×  rF   rG   rW   õ  s     z7AllGatherIntoTensorCoalesced.create.<locals>.<listcomp>c                    s   | d  ˆ 9  < d S rÄ   rF   r.  r/  rF   rG   r0  ÷  s    z9AllGatherIntoTensorCoalesced.create.<locals>.compute_sizer   r1  )r  rv  rˆ   r8  )	ré   r­  r  r  r  r0  r  rd  rA  rF   ©ré   r  rG   rè   í  s    üz#AllGatherIntoTensorCoalesced.createc              
   C   s&   |  |› d|› d|› d|› d�¡ d S )NzO_work = fun_col_impl._all_gather_into_tensor_coalesced_fallback(output_tensors=ú, input_tensors=z_inputs, group=r&  r4  r  rF   rF   rG   r    s    ÿz/AllGatherIntoTensorCoalesced.codegen_collective©r’   r“   r”   r�   rú   r
   rP   rC  rè   r  rû   rF   rF   rß   rG   r8  é  s   ûr8  c                       sF   e Zd Z‡ fdd„Zeed eeee edœdd„ƒZdd„ Z	‡  Z
S )	ÚReduceScatterTensorCoalescedc                    s   t ƒ  ||||¡ || _d S rI   r  r6  rß   rF   rG   r�     s    z%ReduceScatterTensorCoalesced.__init__r:   r"  c           
         sV   ‡ fdd„|D ƒ}‡fdd„}ˆ   ||¡}t|d  ¡ ƒ}t|||||ˆg|d�}	|S )Nc                    s   g | ]}ˆ   |¡‘qS rF   rÖ  r‚  r×  rF   rG   rW      s     z7ReduceScatterTensorCoalesced.create.<locals>.<listcomp>c                    s   | d  ˆ   < d S rÄ   rF   r.  r/  rF   rG   r0  "  s    z9ReduceScatterTensorCoalesced.create.<locals>.compute_sizer   r7  )r  rv  rˆ   r<  )
ré   r­  r   r  r  r  r0  r  rd  rv   rF   r9  rG   rè     s    	ûz#ReduceScatterTensorCoalesced.createc                 C   s2   |  |› d|› d|› dt| jƒ› d|› d�
¡ d S )NzN_work = fun_col_impl._reduce_scatter_tensor_coalesced_fallback(output_tensors=r:  z,_inputs, op=fun_col_impl._str_to_reduce_op('r%  r&  r'  r  rF   rF   rG   r  3  s    &ÿz/ReduceScatterTensorCoalesced.codegen_collectiver;  rF   rF   rß   rG   r<    s   úr<  )T)TFN)FN)Ár¸  rÍ   r  r„  r†  Úloggingrü  Útextwrapr¯   r   Úenumr   r   Úinspectr   Útypingr   r   r   r	   r
   r   r   r   r   r   Zunittest.mockr   r;   r   r   Ztorch._loggingr7   Ztorch.fxZtorch.utils._pytreeÚutilsZ_pytreerÑ  Ztorch._dynamo.utilsr   Ztorch._prims_commonr   r   r   r   r   Ztorch.fx.operator_schemasr   Ztorch.utils._sympy.functionsr   r   r   r³   r   r   Zcodegen.commonr    Zcuda_propertiesr!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   Zvirtualizedr0   r1   Ú	getLoggerr’   r¦  r·   r+  rH   rQ   rb   rd   rh   rê  rs   rC  ry   rŒ   r�   r•   r˜   rš   rœ   r�   rõ  r    r¡   r¥   r§   r8  r¨   rÑ   r   r  r  r  r  r7  r8  r9  r½  r¾  r…   rÕ  rÑ  r  rß  rà  r8   r  r
  r  r¶  rÝ  r;  rA  rC  rE  r-  r�  rõ   rf  ri  rÚ  r|  r  r~  r€  rk  rª  r«  r¾  r¿  r»  r
  r  r  r  r  r  r!  r9   r$  rv  r?  r“  rY  r[  r\  rk  rt  rw  r|  r€  rƒ  r†  r™  rª  r±  rm  r:   rÙ  rÏ   ZInterpreterr¿  rž  rÍ  rÿ  r  r  r  r
  r  r  r  r(  r-  r5  r8  r<  rF   rF   rF   rG   Ú<module>   sŒ  08
$	

@Oú
2    5 }
 ÿJ:): @1m#[Hh s31g  j$"V$% e*  öö ü8H[_5@EVn 	 #o(ld ,55&()