U
    ªÊ¹j¿E  ã                   @   s&  d dl Z d dlZd dl mZ d dlmZmZmZmZmZm	Z	 d dl
mZmZmZmZ ddlmZ d dlmZ d dlZd dlZd dlmZmZmZmZmZ eeef Zeeeed	f f Zd
d„ Ze	ee  e	edœdd„Zeeeed	f f edœdd„Zeeeg e f edœdd„Z!eeeeee	e e	e ef dœdd„Z"e	e e	e eedœdd„Z#dd„ Z$eeeed	f f eeeeedœdd„Z%d d!„ Z&eedd"œd#d$„Z'ed%œd&d'„Z(d(a)da*d)d*„ Z+d+d,„ Z,d-d.„ Z-d/d0„ Z.d1d2„ Z/d3d4„ Z0d5d6„ Z1d7d8„ Z2ed9d:„ ƒZ3ed;d<„ ƒZ4d=d>„ Z5d?d@„ Z6dS )Aé    N)ÚTensor)ÚAnyÚCallableÚOptionalÚTupleÚUnionÚList)Útree_flattenÚtree_unflattenÚ_broadcast_to_and_flattenÚTreeSpecé   )Ú	tree_map_)Úpartial)Ú_add_batch_dimÚ_remove_batch_dimÚ_vmap_decrement_nestingÚ_vmap_increment_nestingÚis_batchedtensor.c                    s    d‰t  ˆ ¡‡ ‡fdd„ƒ}|S )Nzdtorch.func transforms don't yet support saved tensor hooks. Please open an issue with your use case.c               
      s0   t jj ˆ¡� ˆ | |ŽW  5 Q R £ S Q R X d S ©N)ÚtorchZautogradÚgraphZdisable_saved_tensors_hooks)ÚargsÚkwargs©ÚfÚmessage© úT/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/_functorch/vmap.pyÚfn#   s    z.doesnt_support_saved_tensors_hooks.<locals>.fn)Ú	functoolsÚwraps)r   r   r   r   r   Ú"doesnt_support_saved_tensors_hooks   s
    ÿr"   )Úflat_in_dimsÚ	flat_argsÚreturnc                    sZ   dd„ t | |ƒD ƒ‰ tˆ ƒdkr(tdƒ‚ˆ rRt‡ fdd„ˆ D ƒƒrRtdˆ › d�ƒ‚ˆ d S )	Nc                 S   s"   g | ]\}}|d k	r|  |¡‘qS r   )Úsize©Ú.0Úin_dimÚargr   r   r   Ú
<listcomp>.   s    ÿz0_validate_and_get_batch_size.<locals>.<listcomp>r   z/vmap: Expected at least one Tensor to vmap overc                 3   s   | ]}|ˆ d  kV  qdS )r   Nr   )r(   r&   ©Zbatch_sizesr   r   Ú	<genexpr>2   s     z/_validate_and_get_batch_size.<locals>.<genexpr>zTvmap: Expected all tensors to have the same size in the mapped dimension, got sizes z for the mapped dimension)ÚzipÚlenÚ
ValueErrorÚany)r#   r$   r   r,   r   Ú_validate_and_get_batch_size+   s    
ÿr2   )Úbatched_outputsr%   c                 C   s   t | tƒrt| ƒS dS )Nr   )Ú
isinstanceÚtupler/   )r3   r   r   r   Ú_num_outputs9   s    
r6   )ÚvalueÚnum_elementsÚerror_message_lambdar%   c                 C   s.   t | tƒs| f| S t| ƒ|kr*t|ƒ ƒ‚| S r   )r4   r5   r/   r0   )r7   r8   r9   r   r   r   Ú	_as_tupleB   s
    


r:   )Úin_dimsr   Úfuncr%   c           	      C   sÎ  t | tƒs8t | tƒs8tdt|ƒ› d| › dt| ƒ› d�ƒ‚t|ƒdkrXtdt|ƒ› d�ƒ‚t|ƒ\}}t| |ƒ}|d kr¤tdt|ƒ› d| › dt| ƒd › d	|› d�	ƒ‚t	t
||ƒƒD �]\}\}}t |tƒsò|d k	ròtdt|ƒ› d| › d
|› d�ƒ‚t |tƒ�r4t |tƒ�s4tdt|ƒ› d| › d
|› dt|ƒ› d�	ƒ‚|d k	�rš|| ¡  k �s\|| ¡ k�rštdt|ƒ› d| › d
|› d| ¡ › d| ¡ › d| ¡ › d�ƒ‚|d k	r²|dk r²|| ¡  ||< q²t||ƒ|||fS )Núvmap(z
, in_dims=zv, ...)(<inputs>): expected `in_dims` to be int or a (potentially nested) tuple matching the structure of inputs, got: Ú.r   z�)(<inputs>): got no inputs. Maybe you forgot to add inputs, or you are trying to vmap over a function with no inputs. The latter is unsupported.zb, ...)(<inputs>): in_dims is not compatible with the structure of `inputs`. in_dims has structure r   z but inputs has structure z, ...)(<inputs>): Got in_dim=zE for an input but in_dim must be either an integer dimension or None.z' for an input but the input is of type zT. We cannot vmap over non-Tensor arguments, please use None as the respective in_dimz> for some input, but that input is a Tensor of dimensionality z  so expected in_dim to satisfy -z <= in_dim < )r4   Úintr5   r0   Ú	_get_nameÚtyper/   r	   r   Ú	enumerater.   r   Údimr2   )	r;   r   r<   r$   Ú	args_specr#   Úir*   r)   r   r   r   Ú_process_batched_inputsJ   s<    ÿÿ
(ÿÿ$ÿ(8ÿrF   )r#   r$   Ú
vmap_levelr%   c                    s"   ‡ fdd„t | |ƒD ƒ}t||ƒS )Nc                    s(   g | ] \}}|d kr|n
t ||ˆ ƒ‘qS r   )r   r'   ©rG   r   r   r+      s   þz*_create_batched_inputs.<locals>.<listcomp>)r.   r
   )r#   r$   rG   rD   Úbatched_inputsr   rH   r   Ú_create_batched_inputs|   s    
þrJ   c                 C   sp   |d kr6t |tjƒr2t|ƒr2td| › d| › d�ƒ‚|S t |tjƒsbtd| › d| › dt|ƒ› d�ƒ‚t||||ƒS )Nr=   z	, ...): `z5` can not return a BatchedTensor when out_dim is Nonez%` must only return Tensors, got type z2. Did you mean to set out_dim= to None for output?)r4   r   r   r   r0   rA   r   )ÚnameÚbatched_outputrG   Ú
batch_sizeÚout_dimr   r   r   Ú_maybe_remove_batch_dim…   s    ÿ rO   )r3   Úout_dimsrG   rM   r<   r%   c           	         s¬   t | ƒ\}‰‡‡‡fdd„}t| tjƒrntˆtƒr:ˆg}q†tˆtƒrVtˆƒdkrVˆ}q†ˆd krfˆg}q†|ƒ  ntˆˆƒ}|d kr†|ƒ  ‡ ‡‡fdd„t||ƒD ƒ}t	|ˆƒS )Nc                
      s2   t dtˆ ƒ› dˆ› dtˆƒd › dˆ› d�	ƒ‚d S )Nr=   ú, ..., out_dims=z`)(<inputs>): out_dims is not compatible with the structure of `outputs`. out_dims has structure r   z but outputs has structure r>   )r0   r@   r	   r   )r<   rP   Úoutput_specr   r   Úincompatible_errorŸ   s    (ÿz+_unwrap_batched.<locals>.incompatible_errorr   c                    s$   g | ]\}}t tˆƒ|ˆˆ |ƒ‘qS r   )rO   r@   )r(   rL   rN   )rM   r<   rG   r   r   r+   ¶   s   ÿz#_unwrap_batched.<locals>.<listcomp>)
r	   r4   r   r   r?   r5   r/   r   r.   r
   )	r3   rP   rG   rM   r<   Zflat_batched_outputsrS   Úflat_out_dimsZflat_outputsr   )rM   r<   rP   rR   rG   r   Ú_unwrap_batched™   s"    

þrU   c                 C   s8   t | tƒrd S | d krd S tdt|ƒ› d|› d�ƒ‚d S )Nr=   rQ   zˆ): `out_dims` must be an int, None or a python collection of ints representing where in the outputs the vmapped dimension should appear.)r4   r?   r0   r@   )Úxr<   rP   r   r   r   Ú_check_int_or_none½   s    
ÿrW   )rP   r<   r%   c                 C   s&   t | tƒrd S ttt|| d�| ƒ d S )N)r<   rP   )r4   r?   r   r   rW   )rP   r<   r   r   r   Ú$_check_out_dims_is_int_or_int_pytreeÈ   s    
rX   ©r<   c                 C   s   t | dƒr| jS t| ƒS )NÚ__name__)ÚhasattrrZ   ÚreprrY   r   r   r   r@   Î   s    
r@   Fc                     sÒ   t rd S da tj dd¡dkr"ds&d S tj ddd¡addlm	‰  ‡ fd	d
„} | tj
jjjƒ | tj
jjjƒ | tj
jjjƒ | tj
jjjƒ | tj
jjjƒ | tj
jjjƒ | tj
jjjƒ | tj
jjjƒ d S )NTZPYTORCH_JITÚ1ÚatenZIMPLZFuncTorchBatchedr   ©Údecomposition_tablec                    s,   | ˆ krt  | ˆ |  ¡ ntd| › �ƒ‚d S )Nz!could not find decomposition for )ÚVMAP_DECOMPOSITIONS_LIBÚimplÚRuntimeError)Údecompr_   r   r   Ú#_register_python_decomposition_vmapî   s    zElazy_load_decompositions.<locals>._register_python_decomposition_vmap)ÚDECOMPOSITIONS_LOADEDÚosÚenvironÚgetr   ZlibraryÚLibraryra   Ztorch._decompr`   Zopsr^   Zmse_loss_backwardÚdefaultZsmooth_l1_loss_backwardZhuber_loss_backwardZnll_loss_forwardZnll_loss2d_forwardZnll_loss_backwardZnll_loss2d_backwardÚaddr)re   r   r_   r   Úlazy_load_decompositionsÝ   s     rm   c                 O   sh   t ƒ  t|| ƒ t||| ƒ\}}}	}
|d k	rPt|	|||ƒ}t| |||
||f|ŽS t| |||	|
||f|ŽS r   )rm   rX   rF   Ú_get_chunked_inputsÚ_chunked_vmapÚ
_flat_vmap)r<   r;   rP   Ú
randomnessÚ
chunk_sizer   r   rM   r#   r$   rD   Úchunks_flat_argsr   r   r   Ú	vmap_implÿ   s.    
  ÿÿ      ÿÿrt   c                 C   s4   | |  }}|g| }| | }|dkr0|  |¡ |S )Nr   )Úappend)Ztotal_elemsrr   Zn_chunksÚchunk_sizesÚ	remainderr   r   r   Úget_chunk_sizes  s    

rx   c                    sN   |f‰ |d k	r&t ||ƒ}tt |¡ƒ‰ t‡ fdd„t| |ƒD ƒƒ}t|Ž }|S )Nc                 3   s6   | ].\}}|d k	r |j ˆ |d�n|gtˆ ƒ V  qd S ©N)rC   )Ztensor_splitr/   )r(   Útr)   ©Z
split_idxsr   r   r-     s   ÿz&_get_chunked_inputs.<locals>.<genexpr>)rx   r5   Ú	itertoolsÚ
accumulater.   )r$   r#   rM   rr   rv   Zflat_args_chunksrs   r   r{   r   rn     s    
þrn   c                 C   sH   g }d }| D ]&}t |ƒ\}}| |¡ |d kr|}qtt|Ž ƒ}||fS r   )r	   ru   Úlistr.   )Zchunks_output_Zflat_chunks_outputÚarg_specÚoutputÚflat_outputZ	arg_specsÚflat_output_chunksr   r   r   Ú_flatten_chunks_output(  s    
rƒ   c                 C   sX   t | |ƒ}t|ƒt|ƒkst‚g }t|ƒD ](\}}| tj|| |d�¡ d ||< q*|S ry   )r   r/   ÚAssertionErrorrB   ru   r   Úcat)rP   r   r‚   rT   r�   ÚidxrN   r   r   r   Ú_concat_chunked_outputs9  s    

r‡   c                 K   sŠ   g }|dkrt  ¡ nd }|D ]H}	t||	ƒ}
|
dkr4q|d k	rFt  |¡ | t| |
||	|||f|Ž¡ qt|ƒ\}}~t|||ƒ}t||ƒS )NÚsamer   )	r   Zget_rng_stater2   Zset_rng_stateru   rp   rƒ   r‡   r
   )r<   r#   rs   rD   rP   rq   r   Zchunks_outputÚrsr$   rM   r‚   r   r�   r   r   r   ro   G  s2    

      ÿÿÿro   c                 C   s   | dkrt d| › �ƒ‚d S )N)ÚerrorZ	differentrˆ   zLOnly allowed values for randomness are 'error', 'different', or 'same'. Got )rc   )rq   r   r   r   Ú_check_randomness_argq  s    r‹   c                 K   sD   t ||ƒ}z,t||||ƒ}	| |	|Ž}
t|
|||| ƒW ¢S tƒ  X d S r   )r   r   rJ   rU   )r<   rM   r#   r$   rD   rP   rq   r   rG   rI   r3   r   r   r   rp   v  s    

rp   c                    s   ‡ ‡‡‡fdd„}|S )Nc                     s<   t ˆ ˆƒ}z$t| ˆ|ƒ}ˆ||Ž}t||ƒW ¢S tƒ  X d S r   )r   r   Úwrap_batchedÚunwrap_batched)r   r   rG   rI   r3   ©rM   r<   r;   rq   r   r   Úinner—  s    

zrestore_vmap.<locals>.innerr   )r<   r;   rM   rq   r�   r   rŽ   r   Úrestore_vmap•  s    r�   c                 C   s4   t | ƒ\}}t||ƒ}|d k	s"t‚t||||ƒ}|S r   )r	   r   r„   rJ   )r   ÚbdimsÚlevelr$   ÚspecZ
flat_bdimsÚresultr   r   r   rŒ   ¢  s
    
rŒ   c                    sR   t | ƒ\}}t|ƒdkr | dfS ‡ fdd„|D ƒ}t|Ž \}}t||ƒt||ƒfS )Nr   r   c                    s0   g | ](}t |tjƒr$tjj |ˆ ¡n|d f‘qS r   )r4   r   r   Z_CZ
_functorchrU   )r(   r*   ©r’   r   r   r+   ®  s   ÿz"unwrap_batched.<locals>.<listcomp>)r	   r/   r.   r
   )r   r’   r$   r“   r”   r€   r‘   r   r•   r   r�   ª  s    
ÿr�   )7r   r    r   Útypingr   r   r   r   r   r   Ztorch.utils._pytreer	   r
   r   r   Zpytree_hacksr   r   rg   r|   Ztorch._C._functorchr   r   r   r   r   r?   Z	in_dims_tZ
out_dims_tr"   r2   r6   Ústrr:   rF   rJ   rO   rU   rW   rX   r@   rf   ra   rm   rt   rx   rn   rƒ   r‡   ro   r‹   rp   r�   rŒ   r�   r   r   r   r   Ú<module>   sn    
 þ 		  þ3   ÿ	   ý$
"	*

