U
    ªÊ¹j¥9  ã                	   @   sò  d dl Z d dlm  mZ d dlmZmZmZ d dl	m
Z
 d dlmZmZ d dlmZmZmZ d dlmZmZmZ d dlmZ d dlmZ d d	lmZmZmZmZ d d
lm Z  d dl!m"Z"m#Z# G dd„ deƒZ$e$ddd�Z%eddd�Z&edddi d d dd�Z'dd„ Z(dd„ Z)G dd„ de j*j+ƒZ,dd„ Z-dd„ Z.dd„ Z/e& 0ej1¡d d!„ ƒZ2e& 0ej3¡d"d#„ ƒZ4e& 0e¡d$d%„ ƒZ5e& 0e¡d&d'„ ƒZ6e& 0ej7¡d(d)„ ƒZ8e& 0e j9j:j;j7¡d*d+„ ƒZ<e& =ej>¡ e& =ej?¡ e& =ej@¡ e& =ejA¡ e& =ejB¡ dS ),é    N)Ú_ExcludeDispatchKeyGuardÚDispatchKeyÚDispatchKeySet)Úsuspend_functionalization)Ú	AOTConfigÚcreate_joint)Ú#_unwrap_all_tensors_from_functionalÚ_wrap_all_tensors_to_functionalÚfunctionalize)Ú!_has_potential_branch_input_aliasÚ$_has_potential_branch_input_mutationÚ!UnsupportedAliasMutationException)ÚHigherOrderOperator)ÚFakeTensorMode)Údisable_proxy_modes_tracingÚmake_fxÚProxyTorchDispatchModeÚtrack_tensor_tree)ÚStorageWeakRef)Ú_get_current_dispatch_modeÚ_pop_mode_temporarilyc                   @   s   e Zd Zdd„ ZdS )Ú
MapWrapperc                 G   s   t |f|žŽ S ©N)Úmap_wrapper)ÚselfÚxsÚargs© r   úZ/var/www/html/bwave/python/venv/lib/python3.8/site-packages/functorch/experimental/_map.pyÚ__call__#   s    zMapWrapper.__call__N)Ú__name__Ú
__module__Ú__qualname__r   r   r   r   r   r   "   s   r   ÚmapT)Z_deprecated_global_nsÚmap_implF)Zfw_compilerZbw_compilerZpartition_fnZdecompositionsZnum_params_buffersZaot_idZkeep_inference_input_mutationsc              
      s  |d ˆ… }|ˆd … }t ƒ �ð tƒ �œ dd„ ‰‡fdd„t|ƒd D ƒ}‡fdd„|D ƒ}t ˆˆ ||žŽ ¡}tdd„ |D ƒƒrœtd	d
d„ |D ƒ› d�ƒ‚‡fdd„|D ƒ}tˆ ƒ||žŽ }	W 5 Q R X ‡ ‡‡fdd„}
t|ƒt|ƒ ‰t|
ƒ|||žŽ }|	|fW  5 Q R £ S Q R X d S )Nc                 S   sB   t | tjƒr>| jtjkr6tj|  ¡ |  ¡ | j| jd�S |  	¡ S | S )N)ÚdtypeÚrequires_grad)
Ú
isinstanceÚtorchÚTensorr%   ÚboolZempty_stridedÚsizeZstrider&   Úclone©Útr   r   r   Úfrom_funK   s    üz$create_fw_bw_graph.<locals>.from_func                    s   g | ]}ˆ |ƒ‘qS r   r   ©Ú.0r   ©r/   r   r   Ú
<listcomp>X   s     z&create_fw_bw_graph.<locals>.<listcomp>r   c                    s$   g | ]}t |tjƒrˆ |ƒn|‘qS r   ©r'   r(   r)   ©r1   Úargr2   r   r   r3   Y   s   ÿc                 s   s$   | ]}|d k	rt |tjƒ V  qd S r   r4   ©r1   Úoutr   r   r   Ú	<genexpr>`   s   þz%create_fw_bw_graph.<locals>.<genexpr>z?Expect outputs of map only contains tensors or None. Got types c                 S   s   g | ]}t |ƒ‘qS r   )Útyper7   r   r   r   r3   f   s     Ú.c                    s   g | ]}ˆ |ƒ‘qS r   r   r7   r2   r   r   r3   i   s     c            
         s’   | d ˆ… }| ˆd … }|d ˆ… }|ˆd … }‡fdd„}t |td�}|t|ƒt|ƒ dd„ |D ƒƒ\}}dd„ | D ƒ‰ ‡ fdd	„}	t |	|¡S )
Nc                     s   ˆ | Ž }|dd„ |D ƒfS )Nc                 S   s&   g | ]}t |tjƒr|jrd nd‘qS )TF)r'   r(   r)   r&   )r1   Úretr   r   r   r3   v   s   þ
ÿÿzNcreate_fw_bw_graph.<locals>.joint_f.<locals>.fw_with_masks.<locals>.<listcomp>r   )r   Zfw_out)Úfr   r   Úfw_with_maskst   s    üz:create_fw_bw_graph.<locals>.joint_f.<locals>.fw_with_masks)Z
aot_configc                 S   s   g | ]}|d k	r|j r|‘qS r   )r&   )r1   Zgradr   r   r   r3   €   s    þz7create_fw_bw_graph.<locals>.joint_f.<locals>.<listcomp>c                 S   s$   h | ]}t |tjƒrt| ¡ ƒ’qS r   )r'   r(   r)   r   Ú_typed_storager5   r   r   r   Ú	<setcomp>‰   s   þz6create_fw_bw_graph.<locals>.joint_f.<locals>.<setcomp>c                    s(   t | tjƒr$t|  ¡ ƒˆ kr$|  ¡ S | S r   )r'   r(   r)   r   r?   r,   r-   ©Zinput_storager   r   Úmaybe_clone�   s    
ÿþz8create_fw_bw_graph.<locals>.joint_f.<locals>.maybe_clone)r   Údummy_aot_configÚlistÚpytreeÚtree_map)
Zexample_argsZjoint_mapped_argsr   Zmapped_inputZmapped_gradsr>   ZjointÚ_ÚgradsrB   )r=   Újoint_num_mappedÚnum_mapped_argsrA   r   Újoint_fm   s"    	þþþz#create_fw_bw_graph.<locals>.joint_f)	r   r   Ú_unstack_pytreerE   rF   ÚanyÚRuntimeErrorr   Úlen)r=   rJ   r   Z	mapped_xsÚpos_argsZ
example_xsZexample_pos_argsZexample_flat_outZexample_gradÚfw_graphrK   Újoint_graphr   )r=   r/   rI   rJ   r   Úcreate_fw_bw_graph5   s2    
þ 
ÿþÿ,rS   c                    s¸   t  |¡\}‰tdd„ |D ƒƒs0td|› d�ƒ‚t|ƒ‰dd„ |D ƒ}|d d ‰ˆdkrbtdƒ‚t‡fd	d„|D ƒƒrˆtd
|› d�ƒ‚d ‰‡ ‡‡‡fdd„}t  t|ˆf||žŽ ˆ¡S )Nc                 s   s   | ]}t |tjƒV  qd S r   r4   )r1   r.   r   r   r   r9       s     zmap_wrapper.<locals>.<genexpr>z.Mapped xs can only consist of tensors. Got xs r;   c                 S   s   g | ]
}|j ‘qS r   ©Úshaper0   r   r   r   r3   ¤   s     zmap_wrapper.<locals>.<listcomp>r   z,Leading dimensions of mapped xs cannot be 0.c                 3   s   | ]}|d  ˆ kV  qdS ©r   Nr   )r1   Z	cur_shape©Úleading_dim_sizer   r   r9   ©   s     z?Leading dimensions of mapped xs must be consistent. Got shapes c                     s@   t  | d ˆ… ˆ¡}ˆ |f| ˆd … žŽ }t  |¡\}}|‰|S r   )rE   Útree_unflattenÚtree_flatten)Ú	flat_argsr   Zunflattened_outÚflat_outZtmp_out_spec)r=   rJ   Úout_specÚxs_specr   r   Úflat_fn°   s
    zmap_wrapper.<locals>.flat_fn)rE   rZ   ÚallrN   rO   rM   rY   r$   )r=   r   r   Úflat_xsZshapesr_   r   )r=   rX   rJ   r]   r^   r   r   ž   s$    
ÿ	 ÿr   c                   @   s$   e Zd Zedd„ ƒZedd„ ƒZdS )ÚMapAutogradOpc              
   G   sJ   | j |Ž  || _|| _tj ¡ �  t||f|žŽ ˜W  5 Q R £ S Q R X d S r   )Zsave_for_backwardÚ_joint_graphÚ_num_mapped_argsr(   Ú_CZ_AutoDispatchBelowAutogradr$   )ÚctxrQ   rR   rJ   r[   r   r   r   Úforward¿   s
    
zMapAutogradOp.forwardc                 G   sJ   | j }|d | j… }|| jd … }t| j| jt|ƒ f|||žŽ }d|˜S )N)NNN)Zsaved_tensorsrd   r$   rc   rO   )rf   Z
flat_gradsZfw_argsZfw_mapped_argsrP   rH   r   r   r   ÚbackwardÇ   s    þûzMapAutogradOp.backwardN)r    r!   r"   Ústaticmethodrg   rh   r   r   r   r   rb   ¾   s   
rb   c              	      s  t |d |… ƒ}t ||d … ƒ}|d jd ‰ t|ƒd }|}t|tjjƒs\t|ƒ||žŽ }tƒ �* |||žŽ }	‡ fdd„}
t	 
|
|	¡}W 5 Q R X d }d}|sÆd|› �}t| jj|ƒrÀ|d7 }qš|}qš| jj ||¡ ||f|˜}t	 
| jj|¡}| jjd||i dd�}t||d | jd	�S )
Nr   c                    s"   t | tjƒr| jˆ f| jžŽ S | S r   )r'   r(   r)   ÚexpandrU   r-   rW   r   r   Úexpand_tensorä   s    z trace_map.<locals>.expand_tensorZbody_graph_é   Zcall_functionr$   )Úname)ZconstantÚtracer)rD   rU   rL   r'   r(   ZfxZGraphModuler   r   rE   rF   Úhasattrrn   ÚrootZregister_moduleZunwrap_proxyZcreate_proxyr   )Z
proxy_modeZfunc_overloadr=   Ú
num_mappedr   r   rP   Zexample_inputZ
body_graphZexample_outsrk   Zexpanded_outsZ	next_nameÚiÚ	candidateZ	node_argsZ
proxy_argsZ	out_proxyr   rW   r   Ú	trace_map×   sD    

    ÿ   ÿrt   c                    sˆ   t  | ¡\‰ }tdd„ ˆ D ƒƒs.tdˆ › �ƒ‚t‡ fdd„ˆ D ƒƒs\tddd„ ˆ D ƒ› �ƒ‚tˆ Ž }g }|D ]}| t  ||¡¡ ql|S )Nc                 s   s   | ]}t |tjƒV  qd S r   r4   r0   r   r   r   r9     s     z"_unstack_pytree.<locals>.<genexpr>zLeaves of xs must be Tensor c                 3   s&   | ]}|j d  ˆ d  j d  kV  qdS rV   rT   r0   ©ra   r   r   r9     s     z3Leaves of xs must have same leading dimension size c                 S   s   g | ]
}|j ‘qS r   rT   r0   r   r   r   r3     s     z#_unstack_pytree.<locals>.<listcomp>)rE   rZ   r`   rN   ÚzipÚappendrY   )r   ZinspecÚaÚpytreesÚtupler   ru   r   rL   ÿ   s    ÿrL   c                 C   sž   g }d }| D ]}t  |¡\}}| |¡ qt|Ž }g }|D ]V}tdd„ |D ƒƒrb| t |¡¡ q:tdd„ |D ƒƒr€| d ¡ q:td|› d�ƒ‚q:t  ||¡S )Nc                 s   s   | ]}t |tjƒV  qd S r   r4   ©r1   Úleafr   r   r   r9     s     z _stack_pytree.<locals>.<genexpr>c                 s   s   | ]}|d kV  qd S r   r   r{   r   r   r   r9     s     zCannot stack r;   )	rE   rZ   rw   rv   r`   r(   ÚstackrN   rY   )ry   r\   r]   ÚptZflat_ptÚbZstacked_outÚleavesr   r   r   Ú_stack_pytree  s    r�   c                 G   sD   |d |… }||d … }g }t |ƒD ]}| | ||žŽ ¡ q$t|ƒS r   )rL   rw   r�   )r=   rJ   r   r   rP   ry   Zinpr   r   r   Ú	map_dense%  s    r‚   c                 G   s,   t | |f|žŽ \}}tj|||f|žŽ }|S r   )rS   rb   Úapply)r=   rJ   r   rQ   Zbw_graphr\   r   r   r   Úmap_autograd/  s    r„   c              
   G   sn   t ƒ }|d k	stdƒ‚tƒ �H}|jrDt|t| |f|žŽ W  5 Q R £ S t| |f|žŽ W  5 Q R £ S W 5 Q R X d S )Nz5Mode should always be enabled for python fallback key)r   ÚAssertionErrorr   Zenable_tracingrt   r$   )r=   rq   r   Úmoder   r   r   Úmap_proxy_torch_dispatch_mode6  s     r‡   c                 G   s   t | |f|žŽ S r   )r‚   )r=   rq   r   r   r   r   Úmap_fake_tensor_modeA  s    rˆ   c              
   G   sÜ   t j ¡ }|d |… }||d … }t||d�}t||d�}|rBdnd}tttjƒƒ�~ t| |d�}	t	ƒ � t
|ƒd |˜}
W 5 Q R X t| |
ƒr–tdƒ‚t| |
ƒr¨tdƒ‚t|	|f||žŽ }t|dd�W  5 Q R £ S Q R X d S )	N©Úreapply_viewsÚmutations_and_viewsÚ	mutations©Úremover   ú torch.map is mutating the input!ú torch.map is aliasing the input!©Úlevel)r(   re   Z$_functionalization_reapply_views_tlsr   r   r   r   ÚFunctionalizer
   r   rL   r   r   r   r$   r	   )r=   rq   r   rŠ   r   rP   Úunwrapped_xsÚunwrapped_argsr†   Úfunctional_map_fnÚexample_inputsÚ
map_returnr   r   r   Úmap_funcF  s2    
 ÿ

 ÿ ÿr™   c              
   G   sØ   |d|… }||d… }|   ¡ }|r(dnd}t||d�}t||d�}	t||d�}
|  ¡ �v tƒ � t|ƒd |	˜}W 5 Q R X t||ƒrŽtdƒ‚t||ƒr tdƒ‚t	|
|f||	žŽ }t
||  ¡ d	�W  5 Q R £ S Q R X dS )
zÀ
    Functionalization implementation for torch.map. Currently:
      1. We don't allow any input mutation inside the map function
      2. Our check for above condition is not exhaustive
    Nr‹   rŒ   r‰   r�   r   r�   r�   r‘   )Zfunctionalize_add_back_viewsr   r
   Úlowerr   rL   r   r   r   r$   r	   r’   )Úinterpreterr=   rq   r   r   rP   rŠ   r†   r”   r•   r–   r—   r˜   r   r   r   Úmap_functionalizeb  s2     ÿ


 ÿ ÿrœ   )Cr(   Ztorch.utils._pytreeÚutilsZ_pytreerE   Ztorch._Cr   r   r   Ztorch._dispatch.pythonr   Ztorch._functorch.aot_autogradr   r   Z!torch._functorch.eager_transformsr   r	   r
   Ztorch._higher_order_ops.condr   r   r   Z
torch._opsr   Ztorch._subclasses.fake_tensorr   Z"torch.fx.experimental.proxy_tensorr   r   r   r   Z torch.multiprocessing.reductionsr   Ztorch.utils._python_dispatchr   r   r   r#   r$   rC   rS   r   ZautogradÚFunctionrb   rt   rL   r�   Zpy_implZCompositeExplicitAutogradr‚   ZAutogradr„   r‡   rˆ   r“   r™   re   Z
_functorchZTransformTyperœ   ZfallthroughZPythonDispatcherZPythonTLSSnapshotZADInplaceOrViewZBackendSelectZAutocastCPUr   r   r   r   Ú<module>   s\   ùi (

	







"