U
    ªÊ¹jO ã                   @   sô  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZmZmZmZ zd dlZW n ek
rœ   dZY nX d dlZd dlmZ d dlmZmZ d dlmZ d dlmZmZ d dlmZm Z m!Z! d dl"m#Z# d d	l$m%Z% d d
l&m'Z'm(Z( ddl)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/m0Z0m1Z1m2Z2 ddl3m4Z4 ddl5m6Z6m7Z7 ddl8m9Z9 ddl:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZD ddlEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZV ddlWmXZXmYZYmZZZ ddl[m\Z\ ddl]m^Z^m_Z_ ddl`maZambZb ddlcmdZdmeZemfZfmgZgmhZh ddlimjZjmkZkmlZl ddlmmnZnmoZompZp ddlqmrZr ddlsmtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z} ddl~mZm€Z€m�Z�m‚Z‚mƒZƒm„Z„m…Z…m†Z†m‡Z‡mˆZˆ ddl‰mŠZŠm‹Z‹ ddlŒm�Z� dd lŽm�Z�m�Z�m‘Z‘m’Z’m“Z“m”Z” dd!lm•Z•m–Z–m—Z— dd"l˜m™Z™mšZšm›Z› e œe�¡ZžeZŸG d#d$„ d$ƒZ ej¡G d%d&„ d&ƒƒZ¢ej¡G d'd(„ d(ƒƒZ£G d)d*„ d*ƒZ¤d+d,„ Z¥d;d-d.„Z¦d<d0d1„Z§ej¡G d2d3„ d3ƒƒZ¨d4d5„ Z©d=eeC eªd6œd7d8„Z«G d9d:„ d:ƒZ¬dS )>é    N)ÚListÚ
NamedTupleÚOptionalÚUnion)ÚSymInt)ÚGuardSourceÚTracingContext)ÚHigherOrderOperator)Ú
FakeTensorÚis_fake)ÚDimConstraintÚ
DimDynamicÚRelaxedUnspecConstraint)Úimmutable_list)Úis_traceable_wrapper_subclass)ÚTensorWeakRefÚ	WeakIdRefé   )ÚconfigÚmutation_guardÚreplay_recordÚ	skipfiles)Ú
is_allowedÚis_builtin_callableÚis_numpyÚis_user_defined_allowed)Úunimplemented)ÚGuardBuilderÚmake_dupe_guard)ÚSideEffects)
Ú
AttrSourceÚConstantSourceÚGetItemSourceÚGlobalWeakRefSourceÚis_constant_sourceÚLocalSourceÚNumpyTensorSourceÚRandomValueSourceÚSourceÚTupleIteratorGetItemSource)Úbuild_checkpoint_variableÚclone_inputÚget_fake_valueÚget_static_address_typeÚgetfileÚglobal_key_nameÚis_namedtupleÚ	is_typingÚis_utils_checkpointÚistypeÚodict_valuesÚpreserve_rng_stateÚtensor_always_has_static_shapeÚtuple_iteratorÚtuple_iterator_getitemÚtuple_iterator_lenÚwrap_fake_exceptioné   )ÚMutableLocalÚtypestrÚVariableTracker)ÚBuiltinVariable)ÚConstantVariableÚEnumVariable)ÚCUDAStreamVariableÚNullContextVariable)ÚConstDictVariableÚDataClassVariableÚDefaultDictVariableÚHFPretrainedConfigVariableÚPythonSysModulesVariable)ÚDeviceMeshVariableÚPlacementClassVariableÚProcessGroupVariable)Ú!CollectiveFunctionRewriteVariableÚUserFunctionVariableÚUserMethodVariable)Ú TorchHigherOrderOperatorVariable)
ÚBaseListVariableÚDequeVariableÚListVariableÚNamedTupleVariableÚRangeVariableÚSetVariableÚSizeVariableÚSliceVariableÚTupleIteratorVariableÚTupleVariable)
ÚAutogradFunctionContextVariableÚAutogradFunctionVariableÚComptimeVariableÚGetAttrVariableÚInspectSignatureVariableÚLambdaVariableÚNumpyVariableÚPythonModuleVariableÚSkipFilesVariableÚTypingVariable)ÚFSDPManagedNNModuleVariableÚUnspecializedNNModuleVariable)ÚOptimizerVariable)ÚNumpyNdarrayVariableÚSymNodeVariableÚTensorSubclassVariableÚTensorVariableÚTensorWithTFOverrideVariableÚUnspecializedPythonVariable)Útensor_dunder_fnsÚtorch_special_class_typesÚTorchVariable)ÚKeyedJaggedTensorVariableÚUserDefinedClassVariableÚUserDefinedObjectVariablec                   @   s   e Zd ZdS )Ú_missingN)Ú__name__Ú
__module__Ú__qualname__© rw   rw   ú^/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/_dynamo/variables/builder.pyrs   ”   s   rs   c                   @   s„   e Zd ZU eed< eeejf ed< e	ed< e
ejjj ed< dZe	ed< dZe
ej ed< ed	d
„ ƒZdd„ Zdd„ Zdd„ ZdS )ÚGraphArgÚsourceÚ_exampleÚis_unspecializedÚfake_tensorTÚ	is_tensorNÚexample_strong_refc                 C   s.   t | jtƒr$|  ¡ }|d k	s t‚|S | jS d S ©N)Ú
isinstancer{   r   ÚAssertionError)ÚselfÚrrw   rw   rx   Úexample¬   s
    zGraphArg.examplec                 C   s,   t | jtjƒr(t| jƒ| _t| jƒs(t‚d S r€   )r�   r{   ÚtorchÚTensorr   r   r}   r‚   ©rƒ   rw   rw   rx   Ú__post_init__µ   s    zGraphArg.__post_init__c                 C   s   | j  |¡S r€   )rz   Zreconstruct©rƒ   Útxrw   rw   rx   Úloadº   s    zGraphArg.loadc                 C   s
   d | _ d S r€   )r{   rˆ   rw   rw   rx   Úerase½   s    zGraphArg.erase)rt   ru   rv   r(   Ú__annotations__r   r   r†   r   Úboolr   Ú_subclassesr}   r
   r~   r   r‡   Úpropertyr…   r‰   rŒ   r�   rw   rw   rw   rx   ry   ˜   s   

ry   c                   @   s*   e Zd ZU ee ed< eee  ed< dS )ÚFrameStateSizeEntryÚscalarÚsizeN)rt   ru   rv   r   ÚintrŽ   r   rw   rw   rw   rx   r’   Á   s   
r’   c                       s,  e Zd ZdZedœ‡ fdd„Zdd„ Zdd„ Zee	 
d	¡d
d„ ƒƒZedd„ ƒZdd„ Zdd„ Zdd„ Zee	 
d	¡dd„ ƒƒZee	 
d	¡dd„ ƒƒZdd„ Zdd„ Zdd„ Zeeeeef dœdd „Zedœd!d"„Zeeef dœd#d$„Z e!j"j#dœd%d&„Z$d'd(„ Z%e!j&dœd)d*„Z'd+d,„ Z(d-d.„ Z)‡  Z*S )/ÚVariableBuilderz3Wrap a python value in a VariableTracker() instance©rz   c                    sH   |d k	st dƒ‚t ¡ d k	s$t dƒ‚tƒ  ¡  || _|| _| ¡ | _d S )NzRConsider SourcelessBuilder for ephemeral objects, usually objects created locally.zExpected active TracingContext)r‚   r   ÚgetÚsuperÚ__init__r‹   rz   Úname)rƒ   r‹   rz   ©Ú	__class__rw   rx   rš   Ê   s    ÿþ
zVariableBuilder.__init__c                 C   s|   || j jjkrB| j jj| }t| j|jƒ}|r>| |  |¡¡}|S |  |¡jf |  	¡ Ž}|  
|¡rx| j jj | j||¡}|S r€   )r‹   ÚoutputÚside_effectsr   rz   Ú
add_guardsÚmake_guardsÚ_wrapÚcloneÚoptionsÚ_can_lift_attrs_to_inputsÚtrack_object_existing)rƒ   ÚvalueZside_effect_resultÚ	dup_guardÚvtrw   rw   rx   Ú__call__Ø   s     ÿ

  ÿzVariableBuilder.__call__c                 C   s   t |ƒttttfkrdS dS )NTF)Útyperj   rk   rr   rg   )rƒ   r©   rw   rw   rx   r¥   è   s    üz)VariableBuilder._can_lift_attrs_to_inputsNc                   C   s   ddhS )Nr   r;   rw   rw   rw   rw   rx   Ú_common_constantsò   s    üz!VariableBuilder._common_constantsc                 C   sH   t | ƒrtjtt| ƒd�S ttttt	tt
jjtt
jjttjtit| ƒ S )N)Z	tuple_cls)r0   Ú	functoolsÚpartialrS   r«   ÚtuplerY   ÚlistrR   r4   r†   ÚnnZParameterListZ
ModuleListÚcollectionsÚdequerQ   ©r§   rw   rw   rx   Ú	list_type   s"          úùzVariableBuilder.list_typec                 C   s   | j S r€   r—   rˆ   rw   rw   rx   Ú
get_source  s    zVariableBuilder.get_sourcec                 C   s   d|   ¡ iS )Nrz   )r¶   rˆ   rw   rw   rx   r¤     s    zVariableBuilder.optionsc                    s6   |   ¡ ‰ tˆ tƒs ˆ  ¡ tjkr$d S ‡ fdd„|D ƒS )Nc                    s   h | ]}ˆ   |¡’qS rw   )Ú
make_guard)Ú.0Úguardr—   rw   rx   Ú	<setcomp>  s     z.VariableBuilder.make_guards.<locals>.<setcomp>)r¶   r�   r!   Úguard_sourcer   ZCONSTANT)rƒ   Úguardsrw   r—   rx   r¡     s    ÿþzVariableBuilder.make_guardsc                 C   sÄ   t jt jjt jjf| jfttt	t
jf| jft| jfttf| jfttttd ƒtt jt jt jf| jfg}tjr€tr€| tj| j f¡ i }|D ]6\}}t!|tƒrž|n|fD ]}||ks´t"‚|||< q¤qˆ|S r€   )#r†   r‡   r±   Ú	Parameterr�   r
   Úwrap_tensorr¯   r°   r4   r²   r³   Úwrap_listliker7   Úwrap_tuple_iteratorÚsliceÚrangeÚwrap_slice_ranger•   Úfloatr�   r«   ÚstrÚSizeZdeviceZdtypeÚwrap_literalr   Ztrace_numpyÚnpÚappendÚndarrayÚwrap_numpy_ndarrayr�   r‚   )ÚclsÚentriesÚresultÚtsÚfnÚtrw   rw   rx   Ú_type_dispatch  s4    þø
õø
zVariableBuilder._type_dispatchc                 C   sˆ   ddl m } tjdd„ f|dd„ ftjdd„ ftdd„ fg}i }|D ]>\}}t|ttfƒr^|n|fD ]}||kstt	‚||t
|ƒ< qdqD|S )Nr   )Úcomptimec                 S   s   t tj| j|  tj¡d�S ©N©rz   r¼   )r_   r^   Úcreaterz   r¡   r   ÚFUNCTION_MATCH©rƒ   r§   rw   rw   rx   Ú<lambda>K  s   
ýz.VariableBuilder._id_dispatch.<locals>.<lambda>c                 S   s   t ƒ S r€   )r\   rØ   rw   rw   rx   rÙ   Q  ó    c                 S   s   t t| j|  tj¡d�S rÔ   )r_   Ú_dataclasses_fields_lambdarz   r¡   r   r×   rØ   rw   rw   rx   rÙ   T  s   
ýc                 S   s   t || j|  tj¡d�S rÔ   )ro   rz   r¡   r   r×   rØ   rw   rw   rx   rÙ   \  s   
ý)rÓ   ÚinspectÚ	signatureÚdataclassesÚfieldsrm   r�   r¯   r°   r‚   Úid)rÌ   rÓ   rÍ   rÎ   rÏ   rÐ   rÑ   rw   rw   rx   Ú_id_dispatchC  s$    þ
þ	þîzVariableBuilder._id_dispatchc              	      s„  ˆj }ˆ ¡  tˆƒ¡}|d k	r*|ˆˆƒS ˆ ¡  tˆƒ¡}|d k	rN|ˆˆƒS t ˆdˆ¡‰tˆƒspt	ˆt
jƒrzˆ ˆ¡S tˆƒrŒˆ ˆ¡S ˆtjkr¢tˆjd�S t	ˆttjtjfƒ�r¨t‡fdd„ˆ ¡ D ƒƒ�r¨ˆsòˆ ¡  ¡ ròˆ  tj¡‰ nˆ  tj¡‰ ˆ ¡ D ]$}ˆ |¡�rˆj  t!|ƒ|¡ �q‡fdd„‰‡ ‡‡‡fdd„ˆ ¡ D ƒ}t	ˆtjƒ�r€t"|tˆƒˆ #ˆj$¡ˆ d	�}nt%|tˆƒˆ d	�}ˆjj&j' (ˆjˆ|¡S t)ˆt*j+j,ƒ�rÂˆ -ˆ¡S t. /ˆ¡�rØˆ 0ˆ¡S t	ˆt1ƒ�rtd
d„ ˆD ƒƒ�rt.ˆˆj|tj2ƒd�S t)ˆt3j4ƒ�r2t5ˆˆj|tj2ƒd�S t6ˆƒ�rRt7ˆˆj|tj8ƒd�S t9ˆƒ�rht:ˆjd�S t;ˆƒ�rœt<ˆƒ�r†dˆjj&_=t>ˆˆj|tj?ƒd�S t@ˆƒ�r¼tAˆˆj|tj2ƒd�S tBˆƒ�rötC�sÐtD‚tEˆˆj|tFˆƒ�rêtj?ntjGƒd�S t	ˆttHjIfƒ�rBtJjKtLˆƒdd��rBt ˆdd¡�sBtMˆˆj|tj?ƒd�S tN Oˆ¡�r€tN Pˆ¡\}}ˆj}	|ˆ_tN|ˆ|	||tj?ƒd�S t	ˆtHjIt*jQjRfƒ�r¬tSˆˆj|tj?ƒd�S t	ˆtHjTtUjVfƒ�rÖtWˆˆj|tjXƒd�S t	ˆt*jYjZj[ƒ�rþt\ˆˆj|tj?ƒd�S t)ˆt*jYjZj]ƒ�r8ˆjj&j' ^ˆjˆt_ˆˆj|tjGƒd�¡S t)ˆtHj`ƒ�r¤t	taˆdd ƒt*jYjZj[ƒ�r¤taˆddƒdk�r¤ˆtaˆjbdd ƒk�r¤tct\ˆjbˆj|tj?ƒd�dƒS tC�rÂt)ˆtCjdƒ�rÂˆ eˆ¡S tf gˆ¡�rætf hˆˆ¡ i|tjGƒ¡S tj gˆ¡�rtjˆ|tjGƒd	�S t)ˆtkƒ�r.tljmˆˆjˆ  tjGtjn¡d�S tˆƒjodk�r^t)ˆjbtpƒ�r^t>ˆ|tj?ƒd	�S t)ˆt*jqjrjsƒ�r|ttdƒ �nt)ˆt*jujvƒ�r¦ˆt
jk�r¦twˆˆjd�S t)ˆtHj`ƒ�rt)ˆjbt*j+j,ƒ�rtxˆjtyˆjdƒd�ˆjbƒ}
|
�rôt)|
tzƒ�sütDdƒ‚t{ˆj||
ˆj|tj?ƒd�S t	ˆt}j~ƒ�rLt ˆdd ¡d k�rLtˆj|tj?ƒd�S t€ gˆ¡�r†t€ˆˆjˆ  tjG¡d�}ˆjj&j' ^ˆjˆ|¡S t)ˆt*j�j‚ƒ�r®tƒˆˆjˆ  tjG¡d�S t„ …ˆ¡�rÒt„ˆˆjˆ  tj2¡d�S t† ‡ˆ¡�röt†ˆˆjˆ  tj2¡d�S tˆ ‰ˆ¡�rtˆˆˆj|tj2ƒd�S tŠtˆƒtƒ�r>t‹ˆˆj|tj?ƒd�S tŒˆˆjˆ  tjG¡d�}t� Žtˆƒ¡�sj|S ˆjj&j' ^ˆjˆ|¡S d S )NZ_torchdynamo_inliner—   c                 3   s.   | ]&}t  |¡p$ˆ  |¡p$t|tjƒV  qd S r€   )r@   Ú
is_literalÚtensor_can_be_dict_keyr�   ÚenumÚEnum©r¸   Úkrˆ   rw   rx   Ú	<genexpr>‰  s
   ý
ÿz(VariableBuilder._wrap.<locals>.<genexpr>c                    s   ˆ   | ¡rtt| ƒƒS | S d S r€   )rã   r#   r/   )Úkeyrˆ   rw   rx   Úindex_source   s    
z+VariableBuilder._wrap.<locals>.index_sourcec              	      s6   i | ].}|t ˆjtˆ ¡ ˆ|ƒƒƒˆ| ƒ ˆ ¡“qS rw   ©r–   r‹   r"   r¶   r    ræ   ©r¼   rê   rƒ   r§   rw   rx   Ú
<dictcomp>¦  s   ý  ÿþþz)VariableBuilder._wrap.<locals>.<dictcomp>©r¼   c                 s   s    | ]}t |ƒpt |¡V  qd S r€   )r   r@   râ   ©r¸   Úxrw   rw   rx   rè   ½  s     )r§   rz   r¼   rÕ   T)Zallow_torchF)Zorig_fnZorig_sourcerz   r¼   Ú__self__rt   Ú ÚapplyZbuiltin_function_or_methodz3CUDAStreamVariable does not currently work soundly.z"Failed to produce a valid self objÚenter_result)�r¡   rÒ   r˜   r«   rá   rà   rÜ   Úgetattr_staticr   r3   r   Útraceable_tensor_subclassesr¾   r0   r¿   ÚsysÚmodulesrH   rz   Údictr²   ÚdefaultdictÚOrderedDictÚallÚkeysr¶   Úis_nn_moduler   Z
BOOL_FALSEZ	DICT_KEYSrã   r‹   Zstore_dict_keyr/   rF   r¢   Údefault_factoryrD   rž   rŸ   Z
track_dictr�   r†   r±   ÚModuleÚwrap_moduler@   râ   rÇ   Ú	frozensetÚID_MATCHrä   rå   rA   r   r?   ZBUILTIN_MATCHr2   r*   r   r   Ú!has_user_defined_allowed_in_graphro   r×   r1   rc   r   rÈ   r‚   r`   ÚcallableÚ
TYPE_MATCHÚtypesÚFunctionTyper   Úcheckr.   rb   rL   Zcan_rewriteZrewriteZjitZScriptFunctionrM   Ú
ModuleTyper   ZDummyModulera   ZPYMODULE_MATCHZautogradÚfunctionZFunctionMetar[   ZFunctionCtxr¦   rZ   Ú
MethodTypeÚgetattrrñ   r]   ÚnumberÚwrap_unspecialized_primitiverE   Zis_matching_objectÚwrapr    rG   r	   rO   ÚmakeZ
NAME_MATCHrt   rn   ÚcudaÚstreamsÚStreamr   Ú_CZ_TensorMetari   r–   r    r>   rN   Ú__func__Ú
contextlibÚnullcontextrC   rp   ÚoptimÚ	Optimizerrf   rK   Zis_process_grouprI   Zis_device_meshrJ   Zis_placement_typeÚ
issubclassrq   rr   r   Ú"cls_supports_mutation_side_effects)rƒ   r§   r¡   Ztype_dispatchZid_dispatchré   rÎ   Znew_fnZ
new_sourceZ
old_sourceZself_objrw   rì   rx   r¢   l  s4   


 ÿ


 ÿüþü
ü

ÿýý
ý



ý
ý

ÿ
ýý
ÿþýýûýýý
ýý

ÿ
 ÿþûú	ýú
ÿ ÿ ÿý ÿþÿþ ÿ 
ÿþ ÿþü
ÿþþ
ý
  ÿ
ý
ý
ýýý
ý
  ÿzVariableBuilder._wrapc                 C   s4   t |tjjƒp2t | jtƒo2| jjdko2t | jjtƒS )NÚstate)	r�   r†   r±   r½   rz   r    ÚmemberÚbaser%   rØ   rw   rw   rx   rã   µ  s    
ÿüz&VariableBuilder.tensor_can_be_dict_keyc                 C   s¸   | j o¶t| j tƒo¶t| j jtƒo¶| j jjdko¶t| j jjtƒo¶t| j jjjtƒo¶| j jjjjdko¶t| j jjjjtƒo¶| j jjjjj| j	j
 ¡ kr´t| j	j
| j jjjjj tjjƒS dS )NÚparamsZparam_groupsT)rz   r�   r"   r  Úindexr    r  r%   Z
local_namer‹   Úf_localsrý   r†   r  r  rˆ   rw   rw   rx   Útensor_should_specialize¾  s,    
ÿþýüûúùüþñz(VariableBuilder.tensor_should_specializer´   c                    sx   ˆ  tj¡‰ |D ]}||krtdƒ q‡ ‡fdd„t|ƒD ƒ}ˆ |¡|tƒ ˆ d�}t|tƒrtˆj	j
j ˆj||¡S |S )Nz-list elements are pointing to the list itselfc                    s0   g | ](\}}t ˆjtˆ ¡ |ƒƒ|ƒ ˆ ¡‘qS rw   rë   )r¸   ÚiÚitem©r¼   rƒ   rw   rx   Ú
<listcomp>Û  s   ýÿþz1VariableBuilder.wrap_listlike.<locals>.<listcomp>©Úmutable_localr¼   )r¡   r   ÚLIST_LENGTHr   Ú	enumeraterµ   r<   r3   r°   r‹   rž   rŸ   Z
track_listrz   )rƒ   r§   r%  rž   rÎ   rw   r&  rx   r¿   Ò  s    
ü  ÿ
zVariableBuilder.wrap_listlikec                    s:   ˆ  tj¡‰ ‡ ‡‡fdd„ttˆƒƒD ƒ}t|tƒ ˆ d�S )Nc                    s2   g | ]*}t ˆjtˆ ¡ |ƒƒtˆ|ƒƒ ˆ ¡‘qS rw   )r–   r‹   r)   r¶   r8   r    )r¸   r$  ©r¼   rƒ   r§   rw   rx   r'  ê  s   ýÿþz7VariableBuilder.wrap_tuple_iterator.<locals>.<listcomp>r(  )r¡   r   ZTUPLE_ITERATOR_LENrÂ   r9   rX   r<   )rƒ   r§   rž   rw   r,  rx   rÀ   è  s    
ü  ÿz#VariableBuilder.wrap_tuple_iteratorc                    sJ   ‡ ‡fdd„dD ƒ}t ˆtƒr2t|ˆ  tj¡d�S t|ˆ  tj¡d�S d S )Nc                    s,   g | ]$}t ˆ jtˆ  ¡ |ƒƒtˆ|ƒƒ‘qS rw   )r–   r‹   r    r¶   r  ræ   rØ   rw   rx   r'  õ  s   ýÿz4VariableBuilder.wrap_slice_range.<locals>.<listcomp>)ÚstartÚstopÚsteprî   )r�   rÁ   rW   r¡   r   r  rT   ZEQUALS_MATCH)rƒ   r§   Úitemsrw   rØ   rx   rÃ   ô  s    ü
 
ÿ 
ÿz VariableBuilder.wrap_slice_rangec                 C   s:  ddl m} t||ƒrB|  tj¡}t| jdƒ| _|  |j	¡ 
|¡S t|tjjtjjtjjfƒrltjsltdƒ t |¡r²t||  tj¡d�}t t|ƒ¡sœ|S | jjj | j||¡S t|jtjj j!j"ƒrÚt||  tj¡d�S t#|ddƒ�rt#|ddƒ� sþt$d	ƒ‚t%||  tjtj&¡|  '¡ d
�S | jjj(|| j)|  '¡ d�S d S )Nr   )ÚOptimizedModuleÚ	_orig_modz5TorchDynamo purposely graph breaks on RNN, GRU, LSTMsrî   Z_is_fsdp_managed_moduleFZ_fsdp_use_orig_paramsz3Dynamo only supports FSDP with use_orig_params=True)r¼   rz   r—   )*Z
eval_framer1  r3   r¡   r   r  r    rz   r  r2  r    r�   r†   r±   ZRNNZGRUZLSTMr   Z	allow_rnnr   r   Zis_dynamic_nn_modulere   r   r  r«   r‹   rž   rŸ   r¦   r  r�   ÚparallelÚdistributedZDistributedDataParallelr  r‚   rd   r  r¶   Úregister_attr_or_moduler›   )rƒ   r§   r1  r¼   rÎ   rw   rw   rx   r    s`    
ÿþ
 
ÿ
  ÿ 
ÿ 
ÿ  ÿþýýzVariableBuilder.wrap_modulec                    s°   t j }|r@t|ƒtjkr@t‡ fdd„t|ƒD ƒˆ  tj	¡d�S |r˜t|ƒt
kr˜|ˆ  ¡ ksxˆ j ¡  ¡ rxˆ j ¡  ¡ rŒt|ˆ  tj¡d�S ˆ  |¡S nt|ˆ  tj¡d�S d S )Nc                    s*   g | ]"\}}t ˆ jtˆ  ¡ |ƒƒ|ƒ‘qS rw   )r–   r‹   r"   r¶   )r¸   r$  Úvrˆ   rw   rx   r'  M  s   ÿz0VariableBuilder.wrap_literal.<locals>.<listcomp>rî   ©r§   r¼   )r   Úspecialize_intr«   r†   rÆ   rV   r+  r¡   r   r*  r•   r¬   rz   r»   Úis_localrþ   r@   ÚCONSTANT_MATCHr  )rƒ   r§   Zunspecrw   rˆ   rx   rÇ   I  s0    
þ
û
ÿýù	
þ
þzVariableBuilder.wrap_literalc                 C   s(  |   ¡ }| ¡  ¡ s,t|ƒd k	rB| ¡  ¡ sB| jjj|| j|d�S t	|ƒrj| jjj|t
 dd| j¡|d�S t|ƒtjkr~d}n6t|ƒtjtjjtjjjfks°t|ƒs°tt|ƒƒ‚d}|| jjjk}|rÔ| jjj| S || jjjk�r| jjj| }|  |¡}|�r| |  |¡¡}|S | jjjjt
 dd| j¡t|ƒ|d�}t| j|||  t j!t"j#t$|t%ƒ�rb|nt&|ƒd�¡|  '¡ ||d�}|| jjj|< d|j(j)k�sžt‚|j* +¡ |j(j)d< d }	|j,j(j)d	 }
t-|
ƒ�rÐ|
}	t.||d|	ƒ}||j(j)d
< | jj /|¡ t|ƒtjk�r$t0 1| j|||j2j3t|ƒ¡S |S )Nr—   ú[^a-zA-Z0-9]+Ú_TFr´   )r‹   ÚproxyÚexample_valuer¼   Úshould_specializeÚignore_subclassrz   Ztensor_dictr>  Úgrapharg)4r¶   r»   rþ   r-   Zis_fsdp_moduler‹   rž   r5  r›   r$   ÚreÚsubr«   r   rö   r†   r‡   r±   r½   r�   r}   r
   r   r‚   Úinput_source_to_varZ"real_value_tensor_positive_aliasesZ_make_dupe_guardr    r¡   Úroot_tracerÚcreate_graph_inputÚwrap_fx_proxyr­   r®   r   ZTENSOR_MATCHr�   r&   r   r#  ÚnodeÚmetaÚ__dict__Úcopyr=  r   ry   Zadd_symbol_bindingsrk   rÖ   Z__torch_function__r  )rƒ   r§   rz   r@  Zis_duplicate_tensorZstored_valuer¨   Ztensor_proxyZtensor_variableÚfake_tensor_valuer>  rA  rw   rw   rx   r¾   k  sž    
ÿ
þ
ýýýýüü

  ÿÿüÿò
ûzVariableBuilder.wrap_tensorc           
      C   sÎ   t d k	st‚t|t jƒst‚t|  ¡ ƒ}t |¡}t| j	|ƒ|ƒ}| j	j
jjt dd| j¡t|ƒ|d�}||jdœ}tf t| j	||dœ|—Ž}|| j	j
j|< |jjjd }t||d|d|d�}	|	|jjd	< |S )
Nr;  r<  r—   rÕ   ©Ú
target_clsr‹   r=  r>  r>  T)r|   r}   r~   r   rA  )rÈ   r‚   r�   rÊ   r&   r¶   r†   Z	as_tensorr–   r‹   rž   rE  rF  rB  rC  r›   r«   r¼   Úwrap_fx_proxy_clsrg   rD  r=  rH  rI  ry   )
rƒ   r§   rz   Ztensor_valueZ	tensor_vtr=  r¤   Znumpy_ndarray_variabler>  rA  rw   rw   rx   rË   ñ  s>    

  ÿüûúz"VariableBuilder.wrap_numpy_ndarrayc                 C   s¶  | j | jjjkr | jjj| j  S t|tƒ�r6t|  ¡ ƒ�s6t|  ¡ tƒ�s6t	j
jjrht||  tj¡d�S | jjj}| j  ¡ }|| jjjkr˜t|d d�}n0| jjj| }|j|krÈt d|||j¡ d |_|| jjj|< tjræ|jd ksìtjsôtj}nt||  tj¡d�S |j|| j|d�}| jjj t|| jd ƒ¡ n
t	  |¡}t|  ¡ tƒ�sn|  ¡  !tj"d¡h}d|i}ni }| #d|  ¡ i¡ t|t	j$ƒ�r | #d|i¡ | jjj%j&t' (d	d
| j ¡t)|ƒ|  ¡ d�}	t*t+f| j|	|dœ|—Ž}
|
| jjj| j < t|  ¡ ƒ�s®| jj,�r0t|  ¡ t-ƒ�s0t.d /||  ¡ ¡ƒ‚d }t|
tƒ�rH|
j0}n|
j1j2j3d }t4|ƒ�r†|}|j5| jj5k�s†t.d|j5› d�ƒ‚t6|  ¡ |t|t	j$ƒ|d|d�|	j2j3d< |
S d S )Nr7  )r“   r”   z%automatic dynamic int %s val %s != %s)rz   Údynamic_dimTr¼   rz   Z	raw_valuer;  r<  r—   )r‹   r=  r>  zJDynamo attempts to add additional input during export: value={}, source={}r>  zfake mode (z^) from fake tensor metadata doesn't match mode({self.tx.fake_mode}) from InstructionTranslatorF)r~   r   rA  )7r›   r‹   rž   Zunspec_variable_mapr�   r•   r$   r¶   r'   r†   Z_dynamor   r8  r@   r¡   r   r:  Ú	shape_envrz   Úframe_stater’   r“   ÚlogÚdebugÚautomatic_dynamic_shapesÚassume_static_by_defaultr   ÚDYNAMICZ$create_unspecified_symint_and_symbolÚtracked_fakesrÉ   ÚTrackedFakeÚtensorr·   r  Úupdater‡   rE  rF  rB  rC  r«   rO  rl   Úexportr%   r‚   Úformatr§   r=  rH  rI  r   Ú	fake_modery   )rƒ   r§   rQ  r›   Úframe_state_entryrP  Zwrapped_valuer¼   r¤   r=  Z
unspec_varrL  r>  rw   rw   rx   r    s¾    ÿ
þý

þ


üÿÿþ
þý
ÿ


ýÿüû ÿÿ
ÿ
úz,VariableBuilder.wrap_unspecialized_primitive)+rt   ru   rv   Ú__doc__r(   rš   rª   r¥   Ústaticmethodr­   Ú	lru_cacher¬   rµ   r¶   r¤   r¡   ÚclassmethodrÒ   rá   r¢   rã   r#  r   r¯   r°   r4   r   r¿   r7   rÀ   rÁ   rÂ   rÃ   r†   r±   r   r  rÇ   r‡   r¾   rË   r  Ú__classcell__rw   rw   rœ   rx   r–   Ç   sB   ý

	$'  K	E" (r–   c                 C   sŠ   t | tƒr| j}n t | tƒr$| j}ntd| › �ƒ g }t |¡D ]:}d }| jrbt	t
| jdƒ|jƒ}| t||d� | ¡¡ q@t|ƒ | ¡S )Nz)Dataclass fields handling fails for type Ú__dataclass_fields__r—   )r�   rr   r§   rE   Zuser_clsr   rÞ   rß   rz   r"   r    r›   rÉ   Úadd_optionsrY   )Úobjr§   r0  Úfieldrz   rw   rw   rx   rÛ   ”  s    


 ÿrÛ   c                 K   s   t f t| ||dœ|—ŽS )NrM  )rO  rj   )r‹   r=  r>  r¤   rw   rw   rx   rG  ¦  s    üûrG  Fc                    sJ  dd l ‰ddlm} tˆ|ƒs"t‚d|krH|d d k	rHˆjj |d ¡ d|jj	ksft|jj	d › ƒ‚|}‡‡fdd„‰ ‡ ‡fdd	„}t
ƒ �Ä |d kr¤t|jˆƒ}n¨t|tƒrº|jˆjksÂˆ |ƒrÄnˆt|ˆjƒ�rLˆj�rˆj ¡ � ||ƒ|jj|j< W 5 Q R X || tkd
œ}	d|k�r(|d d k	�s,t‚|d |	d< t|fdˆi|	—Ž}W 5 Q R X t|ˆjƒ�rt|ˆjjƒ}
| dd¡}|
�sŠ|�r�|}nd }||ƒ}||jj	d< |  |¡}t|ˆjjjƒ�rê|jˆjk�rê|
�ràˆjjnˆj|d< ||d< | |¡ | |f|ŽS t|jjdƒ�r>|jjjdk�r>t|jjjˆjj ƒ�sP|jjˆj!j"k�rhddl#m$} ||jjƒS |jjˆjj%k�sŒ|jjˆj&j'k�r ddl#m(} ||ƒS t)|ˆj*ƒ�rÜt+dd„ |D ƒƒ�rÜdd„ |D ƒ}t,|f|ŽS t|t-t.t/fƒ�r(||jj	d< g }t0|ƒD ]X\}}|d k�r*| 1t2d$|Ž¡ n2| 1t3| ˆ|j 4dt5j6||fi ¡fd|i|—Ž¡ �qt|ˆj*ƒ�r|t,||f|ŽS t)|t-ƒ�r”t7|f|ŽS t)|t.t8fƒ�rºt9|fdt:ƒ i|—ŽS t)|t/ƒ�rÞt;ˆ|fdt:ƒ i|—ŽS |j<j=dk�st|dƒ�std|j<j=› dt>|ƒ› �ƒ‚t?||j<f|ŽS �n|d k�sB|jjˆj@k�rLt2d%|ŽS t|ˆjAˆjBˆjCfƒ�r~||jj	d< tD||f|ŽS |jjˆj&jEjFˆj&jGfk�r´||jj	d< tH||f|ŽS t|tIƒ�r|jjˆjJtKt5j6ˆjLjMˆjNt5jOtKˆjPd tQƒtKˆjPd!tQƒˆjRjSjTf	k�r||jj	d< t2|f|ŽS tUd"tV|ƒ› d#|jjW› d#|jj› � ƒ d S )&Nr   r   )ÚInstructionTranslatorBaser¼   r>  c                    sJ   t | ˆ jƒrFˆ  | ¡rFˆ j ¡ }ˆ jj | |¡}t |tƒoD|jˆjkS dS ©NF)	r�   r‡   Z_is_functional_tensorr  Z$_functionalization_reapply_views_tlsZ
_functorchZ_unwrap_functional_tensorr
   r^  )rð   Zreapply_viewsZ	unwrapped)r†   r‹   rw   rx   Ú(_is_functional_tensor_fakified_by_dynamoæ  s    


þzCwrap_fx_proxy_cls.<locals>._is_functional_tensor_fakified_by_dynamoc                    s*   t | ˆjƒr&t | tƒs&ˆ | ƒs&t| ƒ} | S r€   )r�   r‡   r
   r+   r´   )rk  r†   rw   rx   Ú_clone_inputð  s    ÿþz'wrap_fx_proxy_cls.<locals>._clone_input)r@  r~   rz   r‹   r?  FZ
class_typeÚspecialized_valuert   Z	set_stater;   )ro   )rr   c                 s   s   | ]}t |tƒV  qd S r€   )r�   r•   rï   rw   rw   rx   rè   J  s    z$wrap_fx_proxy_cls.<locals>.<genexpr>c                 S   s   g | ]}t |ƒ‘qS rw   )r@   rï   rw   rw   rx   r'  M  s     z%wrap_fx_proxy_cls.<locals>.<listcomp>Zcall_functionr)  ztorch.return_typesÚ_fieldsz	expected z. == torch.return_types or named tuple but got Zget_rankZget_world_sizeztorch.* op returned non-Tensor ú )N)N)XZtorch._export.constraintsZsymbolic_convertri  r�   r‚   rž   r¼   r[  rH  rI  r5   r,   r
   r^  r‡   r\  r  ZDisableTorchFunctionSubclassZtracerZreal_value_cacherj   Úwrap_to_fake_tensor_and_recordr±   r½   ÚpopZ
specializer�   r}   ÚhasattrÚtargetrt   rñ   Ú	GeneratorÚrandomZset_rng_staterò   ro   Z_DisableFuncTorchr  Z_is_in_bad_forkrr   r3   rÆ   rü   rV   r¯   r°   Úsetr+  rÉ   r@   rO  Zcreate_proxyÚoperatorÚgetitemrY   r   rR   r<   rU   r�   ru   r«   rS   Zmanual_seedr   ZSymFloatZSymBoolrh   r  r  Zcurrent_streamrB   r•   Zsym_intr  Z_utilsZ_element_sizeÚseedÚmodr4  rs   Z_exportÚconstraintsZconstrain_as_valuer   r=   Úop)rN  r‹   r=  r>  r@  r¤   ri  Zinitial_example_valuerl  ÚkwargsZis_parameterr?  rm  Zspecialized_propsro   rr   ÚsizesÚunpackedr$  Úvalrw   )rk  r†   r‹   rx   rO  Ø  s   
ÿ
þýÿþÿÿÿ
ÿ
þÿ
ÿþýüÿþÿ

ÿ   ÿýúùÿ ÿþ
ôÿÿrO  c                   @   sT   e Zd ZU eeef ed< eed< ee	e
  ed< edœdd„Zeedœdd	„Zd
S )rY  Úfakerz   Úconstraint_dims©Úreturnc                 C   s   t | j| j ¡ fƒS r€   )Úhashr�  rz   r›   rˆ   rw   rw   rx   Ú__hash__›  s    zTrackedFake.__hash__)Úotherr„  c                 C   s.   t |tƒr*| j|jko(| j ¡ |j ¡ kS dS rj  )r�   rY  r�  rz   r›   )rƒ   r‡  rw   rw   rx   Ú__eq__ž  s    
 zTrackedFake.__eq__N)rt   ru   rv   r   r
   r   rŽ   r(   r   ÚDimListr   r•   r†  Úobjectr�   rˆ  rw   rw   rw   rx   rY  ”  s
   
rY  c              	      s˜  |r"t jg|  ¡  d g|  ¡  fS d }||jjkrLtd d ƒ}t|  ¡ ƒ|_n�|jj| }|jd k	rÜ| jt	|jƒkrŽt
 d|| j|j¡ d |_nNt|jƒD ]B\}}|d k	r˜|  ¡ | |kr˜t
 d|||  |¡|¡ d |j|< q˜t| ƒ}i ‰ ‡ fdd„}|jj�rR|jjD ]J}	|	j|k�r$||	j|	jƒ |	jd k	�r|	jj|k�r||	jj|	jƒ �qg }
g }t|  ¡ ƒD �]}|t| dtƒ ƒk}|t| dtƒ ƒk}|t| dtƒ ƒk}tj�oÂ|jd k�pÂ|j| d k}|j�rè|�rèt
 d|¡ d |j|< ˆ  |¡}	|	d k�r,|�rtj�std	d
�}	n|�s,|�r,tdd
�}	| |	¡ |	d k	�sL|�sL|�rTt j}n"|�shtj�sh|�rpt j}nt j}|
 |¡ �qf||jj|< |
|fS )Nz!automatic dynamic %s dim %s != %sz&automatic dynamic %s size(%s) %s != %sc                    s>   | ˆ kr2ddl m} ||jˆ |  j@ dd�ˆ | < n|ˆ | < d S )Nr   )ÚStrictMinMaxConstraintF)ÚvrÚ	warn_only)Ú%torch.fx.experimental.symbolic_shapesr‹  rŒ  )ÚdimÚconstraint_ranger‹  ©Zdim2constraintrw   rx   Úupdate_dim2constraintÑ  s    þz1_automatic_dynamic.<locals>.update_dim2constraintZ_dynamo_dynamic_indicesZ_dynamo_weak_dynamic_indicesZ_dynamo_static_indicesz#automatic dynamic %s marked dynamicF)r�  T)r   ZSTATICr�  rž   rR  r’   r°   r”   ÚndimÚlenrS  rT  r+  rà   Zexport_constraintsÚt_idr�  ZsharedrÂ   r  rv  r   rU  r˜   Zallow_ignore_mark_dynamicr   rÉ   rW  rV  ZDUCK)Úer‹   r›   Ústatic_shapesr_  r$  r�  r•  r’  Ú
constraintÚdynamic_dimsr‚  Zmarked_dynamicZmarked_weak_dynamicZmarked_staticZautomatic_dynamicZdynamicrw   r‘  rx   Ú_automatic_dynamic¦  s‚    

üû
 ÿÿ




rš  )rz   r~   c                   s  t ˆƒtjtjjtfks0ˆr(tˆtjƒs0tˆƒrüˆd k	s<t‚t	ˆ|ˆ 
¡ d�\}}tˆˆˆ ¡ |ƒ\‰‰ t dˆ ¡ tˆjƒˆˆ ¡ t‡ ‡‡‡‡‡fdd„ƒ}|rÚ|r®ˆ ¡ sÚˆjj t|ˆˆ ƒ¡ ˆjjtˆƒ  ˆ¡ | ¡ | ¡ dœˆjjtˆƒ< |S ˆS d S )N)r»   zwrap_to_fake %s %s %s %sc                      s   ˆj jˆˆˆˆˆ d�S )N)r@  rz   r™  r‚  )r^  Zfrom_tensorrw   ©r‚  r™  r–  r@  rz   r‹   rw   rx   rÙ   4  s   ûz0wrap_to_fake_tensor_and_record.<locals>.<lambda>)r”   Ústride)r«   r†   r‡   r±   r½   r
   r�   r   r‚   r6   r»   rš  r›   rS  rT  r¯   Úshaper:   rþ   rž   rX  rÉ   rY  Ztracked_fakes_id_to_sourcerà   r”   rœ  Ztensor_weakref_to_sizes_stridesr   )r–  r‹   r@  rz   r~   r—  ÚreasonZfake_erw   r›  rx   rp    sL    ÿþ
þý  ÿ
   ÿûÿ	þrp  c                   @   s*   e Zd ZdZedœdd„Zedd„ ƒZdS )ÚSourcelessBuildera½  
    Like builder, but stateless and does not require a source. Useful for simple type->VT objects, or objects
    that are being created/evaporated during inlining (ex: consider a locally made list of tensors we then iterate over
    .), such a list should not show up as an artifact from inputs, nor in reconstruction, nor in the graph. However,
    there may be reasons to represent it as a ListVariable internally.

    NOTE - Objects produced here are born UNGUARDED due to the nature of sources!

    NOTE - This class is very new! It will have some rough edges, but it was created to stem the bleeding of giant
    if/else type->VariableTracker trees that were cropping up all over dynamo.
    rƒ  c                    s(  t |tƒr|S t |tjƒr"t|ƒS t |¡r6t |¡S t	|ƒrFt
|ƒS t|ƒrht|ƒr`dˆ jj_t|ƒS t |tjƒr|t|ƒS t |tjƒr�t|ƒS t |ttjfƒr¨t|ƒS t |tƒrÖt‡ ‡fdd„| ¡ D ƒttƒ d�S t |tt fƒ�rt! "t|ƒ¡}|‡ ‡fdd„|D ƒtƒ d�S t#dt|ƒ› �ƒ d S )NTc                    s   i | ]\}}|ˆ ˆ|ƒ“qS rw   rw   )r¸   rç   r6  rŠ   rw   rx   rí   k  s      z.SourcelessBuilder.__call__.<locals>.<dictcomp>)r)  c                    s   g | ]}ˆ ˆ|ƒ‘qS rw   rw   rï   rŠ   rw   rx   r'  q  s     z.SourcelessBuilder.__call__.<locals>.<listcomp>z&Unexpected type in sourceless builder )$r�   r>   rÞ   Ú_HAS_DEFAULT_FACTORY_CLASSrr   r@   râ   rŸ  Úwrap_constant_literalr   r?   r   r   r‹   rž   r  ro   r  r  rM   rä   rå   rA   r«   ÚabcÚABCMetarq   rù   rD   r0  r<   r¯   r°   rP   Zcls_forr   )rƒ   r‹   r§   rÌ   rw   rŠ   rx   rª   U  s8    




ýzSourcelessBuilder.__call__c                 C   s   t  | ¡st‚t | d�S )Nr´   )r@   râ   r‚   r´   rw   rw   rx   r¡  t  s    z'SourcelessBuilder.wrap_constant_literalN)rt   ru   rv   r`  r>   rª   ra  r¡  rw   rw   rw   rx   rŸ  H  s   rŸ  )N)NF)F)­r¢  r²   r  rÞ   rä   r­   rÜ   Úloggingrw  rB  r÷   r  Útypingr   r   r   r   ÚnumpyrÈ   ÚModuleNotFoundErrorr†   r   Ztorch._guardsr   r   Z
torch._opsr	   Ztorch._subclasses.fake_tensorr
   r   rŽ  r   r   r   Ztorch.fx.immutable_collectionsr   Ztorch.utils._python_dispatchr   Ztorch.utils.weakr   r   rò   r   r   r   r   Zallowed_functionsr   r   r   r   Úexcr   r¼   r   r   rŸ   r   rz   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   Úutilsr*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r  r<   r=   r>   Úbuiltinr?   Zconstantr@   rA   Zctx_managerrB   rC   ZdictsrD   rE   rF   rG   rH   r4  rI   rJ   rK   Z	functionsrL   rM   rN   Zhigher_order_opsrO   ÚlistsrP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   ÚmiscrZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   Z	nn_modulerd   re   Z	optimizerrf   rZ  rg   rh   ri   rj   rk   rl   rm   rn   ro   Zuser_definedrp   rq   rr   Ú	getLoggerrt   rS  r‰  rs   Ú	dataclassry   r’   r–   rÛ   rG  rO  rY  rš  r�   rp  rŸ  rw   rw   rw   rx   Ú<module>   sš   
0L00 
(       T
3   ÿ
 =v ÿ ÿ-