U
    •Ê¹jù:  ã                   @   sà  d dl Z d dlmZ d dlmZ d dlmZmZmZm	Z	m
Z
mZ d dlmZ d dlmZ d dlmZ d dlmZ dgZe jjZd	d
„ Zddœedœdd„Zd5edœdd„Zd6edœdd„Zd7edœdd„Zd8ee ee ee eedœdd„Zddœedœdd„Zdd„ Z edœdd„Z!dd „ Z"ddœedœd!d"„Z#d#d$„ Z$ddœedœd%d&„Z%ej&eej'eej(eej)eej*eej+eej,e!ej-e#ej.e#ej/e%ej0e%iZ1d'd(„ Z2d)d*d+d,d-gZ3d.d/„ Z4d0d1„ Z5G d2d„ deƒZ6G d3d4„ d4eƒZ7dS )9é    N)Útree_map)ÚListÚAnyÚDictÚOptionalÚUnionÚ
NamedTuple)Údefaultdict)ÚTorchDispatchMode)ÚRemovableHandle©ÚprodÚFlopCounterModec                 C   s   t | tjƒr| jS | S ©N)Ú
isinstanceÚtorchÚTensorÚshape)Úi© r   úW/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/utils/flop_counter.pyÚ	get_shape   s    r   )Ú	out_shape©Úreturnc          	      O   s,   | \}}|\}}||kst ‚|| d | S )z!
    Count flops for matmul.
    é   ©ÚAssertionError)	Úa_shapeÚb_shaper   ÚargsÚkwargsÚmÚkÚk2Únr   r   r   Úmm_flop   s    r&   c                 K   s
   t ||ƒS )z
    Count flops for addmm
    )r&   ©Z
self_shaper   r   r   r!   r   r   r   Ú
addmm_flop   s    r(   c                 K   sD   | \}}}|\}}}	||ks t ‚||ks,t ‚|| |	 d | }
|
S )z,
    Count flops for the bmm operation.
    r   r   )r   r   r   r!   Úbr"   r#   Úb2r$   r%   Úflopr   r   r   Úbmm_flop%   s    

r,   c                 K   s
   t ||ƒS )z0
    Count flops for the baddbmm operation.
    )r,   r'   r   r   r   Úbaddbmm_flop3   s    r-   F)Úx_shapeÚw_shaper   Ú
transposedr   c           
      C   sJ   | d }|r| n|dd… }|^}}}|t |ƒ | t |ƒ d | }	|	S )a  
    Count flops for convolution. Note only multiplication is
    counted. Computation for bias are ignored.
    Flops for a transposed convolution are calculated as
    flops = (x_shape[2:] * prod(w_shape) * batch_size).
    Args:
        x_shape (list(int)): The input shape before convolution.
        w_shape (list(int)): The filter shape.
        out_shape (list(int)): The output shape after convolution.
        transposed (bool): is the convolution transposed
    Returns:
        int: the number of flops
    r   r   Nr   )
r.   r/   r   r0   Z
batch_sizeZ
conv_shapeZc_outZc_inZdimsr+   r   r   r   Úconv_flop_count<   s
    
 r1   c          
      O   s   t | |||d�S )z&
    Count flops for convolution.
    )r0   )r1   )
r.   r/   Ú_biasÚ_strideÚ_paddingÚ	_dilationr0   r   r    r!   r   r   r   Ú	conv_flopX   s    r6   c                 C   s    | d | d gt | dd … ƒ S )Né   r   r   )Úlist)r   r   r   r   Útranspose_shape^   s    r9   c                 C   sZ   d}|
d r,t |d ƒ}|t| ||| ƒ7 }|
d rVt |d ƒ}|tt|ƒ| ||ƒ7 }|S ©Nr   r7   )r   r1   r9   )Úgrad_out_shaper.   r/   r2   r3   r4   r5   r0   Z_output_paddingZ_groupsZoutput_maskr   Ú
flop_countZgrad_input_shapeZgrad_weight_shaper   r   r   Úconv_backward_flopa   s    r=   c                 C   s¼   | \}}}}|\}}}	}
|\}}}}||  kr8|krln n0||  krP|krln n||
krl|	|krl||
kspt ‚d}|t|| ||f|| ||	fƒ7 }|t|| ||	f|| |	|fƒ7 }|S )z]
    Count flops for self-attention.
    NB: We can assume that value_shape == key_shape
    r   ©r   r,   )Úquery_shapeÚ	key_shapeÚvalue_shaper)   ÚhÚs_qÚd_qÚ_b2Ú_h2Ús_kÚ_d2Ú_b3Ú_h3Ú_s3Úd_vÚtotal_flopsr   r   r   Úsdpa_flop_county   s    L""rN   c                O   s   t | ||ƒS )z)
    Count flops for self-attention.
    )rN   )r?   r@   rA   r   r    r!   r   r   r   Ú	sdpa_flop‹   s    rO   c                 C   sR  d}|\}}}}|\}	}
}}|\}}}}| \}}}}||	  krR|  krR|kr€n n*||
  krt|  krt|kr€n n||ks„t ‚||krœ||krœ||ks t ‚d}|t|| ||f|| ||fƒ7 }|t|| ||f|| ||fƒ7 }|t|| ||f|| ||fƒ7 }|t|| ||f|| ||fƒ7 }|t|| ||f|| ||fƒ7 }|S )Nr   r>   )r;   r?   r@   rA   rM   r)   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   Z_b4Z_h4Z_s4Z_d4r   r   r   Úsdpa_backward_flop_count“   s    P"""""rP   c                O   s   t | |||ƒS )z2
    Count flops for self-attention backward.
    )rP   )r;   r?   r@   rA   r   r    r!   r   r   r   Úsdpa_backward_flop®   s    rQ   c                 C   s   t | tƒs| fS | S r   )r   Útuple©Úxr   r   r   Únormalize_tupleÂ   s    
rU   Ú ÚKÚMÚBÚTc                 C   s0   t dtttƒd tt| ƒƒd d ƒƒ}t| S )Nr   r7   é   )ÚmaxÚminÚlenÚsuffixesÚstr)ÚnumberÚindexr   r   r   Úget_suffix_strË   s    (rc   c                 C   s&   t  |¡}| d|  d›}|t |  S )Niè  z.3f)r_   rb   )ra   Úsuffixrb   Úvaluer   r   r   Úconvert_num_with_suffixÒ   s    
rf   c                       sÖ   e Zd ZdZd"eeejje	ejj f  e
eeeeef  dœdd„Zdd	„ Zd
d„ Zdd„ Zdd„ Zdd„ Zdd„ Ze
dœdd„Zeeeee
f f dœdd„Zd#dd„Z‡ fdd„Z‡ fdd„Zd$d d!„Z‡  ZS )%r   a¸  
    ``FlopCounterMode`` is a context manager that counts the number of
    flops within its context. It does this using a ``TorchDispatchMode``.

    It also supports hierarchical output by passing a module (or list of modules) to FlopCounterMode on construction.

    Example usage

    .. code-block:: python

        mod = ...
        flop_counter = FlopCounterMode(mod)
        with flop_counter:
            mod.sum().backward()

    Nr   T)ÚmodsÚdepthÚdisplayÚcustom_mappingc                 C   s\   t dd„ ƒ| _|| _dg| _|| _|d kr.i }t|tjjƒrB|g}|| _	i | _
t|–| _d S )Nc                   S   s   t tƒS r   )r	   Úintr   r   r   r   Ú<lambda>ð   ó    z*FlopCounterMode.__init__.<locals>.<lambda>ÚGlobal)r	   Úflop_countsrh   Úparentsri   r   r   ÚnnÚModulerg   Ú_module_to_forward_hook_handlesÚflop_mapping)Úselfrg   rh   ri   rj   r   r   r   Ú__init__ê   s    zFlopCounterMode.__init__c                 C   sŽ   | j d krd S | j D ]t}t|ƒj}t| ¡ ƒ ¡ D ]T\}}|dkrH|}nd ||g¡}| |  |¡¡}| 	|  
|¡¡}t||ƒ| j|< q2qd S )NrV   Ú.)rg   ÚtypeÚ__name__ÚdictZnamed_modulesÚitemsÚjoinZregister_forward_pre_hookÚ_enter_moduleZregister_forward_hookÚ_exit_moduleÚ_ForwardHookHandlesrs   )ru   ÚmodÚprefixÚnameÚmoduleÚforward_pre_hook_handleÚforward_hook_handler   r   r   Ú_register_forward_hooksý   s    


 ÿz'FlopCounterMode._register_forward_hooksc                 C   s6   | j  ¡ D ]}|d  ¡  |d  ¡  q
| j  ¡  d S r:   )rs   ÚvaluesÚremoveÚclear)ru   Zforward_hook_handlesr   r   r   Ú_deregister_forward_hooks  s    z)FlopCounterMode._deregister_forward_hooksc                    s   ‡ ‡fdd„}|S )Nc                    s   t |ƒ}ˆ ˆ ¡|Ž }|S r   )rU   Ú_create_pre_module)rƒ   ÚinputsÚout©r‚   ru   r   r   Úf  s    z(FlopCounterMode._enter_module.<locals>.fr   ©ru   r‚   r�   r   rŽ   r   r}     s    zFlopCounterMode._enter_modulec                    s   ‡ ‡fdd„}|S )Nc                    s   t |ƒ}ˆ ˆ ¡|Ž S r   )rU   Ú_create_post_module)rƒ   rŒ   ÚoutputsrŽ   r   r   r�     s    z'FlopCounterMode._exit_module.<locals>.fr   r�   r   rŽ   r   r~     s    zFlopCounterMode._exit_modulec                    s    G ‡ ‡fdd„dt jjƒ}|jS )Nc                       s0   e Zd Ze‡ ‡fdd„ƒZe‡ ‡fdd„ƒZdS )z6FlopCounterMode._create_post_module.<locals>.PushStatec                    sB   ˆj d ˆ kst‚ˆj  ¡  tdd„ |ƒ}t|ƒdkr>|d S |S )Néÿÿÿÿc                 S   s   t | tjƒr|  ¡ S | S r   ©r   r   r   ÚclonerS   r   r   r   rl   '  rm   zPFlopCounterMode._create_post_module.<locals>.PushState.forward.<locals>.<lambda>r7   r   )rp   r   Úpopr   r^   ©Úctxr    rŽ   r   r   Úforward#  s    
z>FlopCounterMode._create_post_module.<locals>.PushState.forwardc                    s   ˆj  ˆ ¡ |S r   )rp   Úappend©r˜   Z	grad_outsrŽ   r   r   Úbackward,  s    z?FlopCounterMode._create_post_module.<locals>.PushState.backwardN©ry   Ú
__module__Ú__qualname__Ústaticmethodr™   rœ   r   rŽ   r   r   Ú	PushState"  s   r¡   ©r   ZautogradÚFunctionÚapply)ru   r‚   r¡   r   rŽ   r   r‘   !  s    z#FlopCounterMode._create_post_modulec                    s    G ‡ ‡fdd„dt jjƒ}|jS )Nc                       s0   e Zd Ze‡ ‡fdd„ƒZe‡ ‡fdd„ƒZdS )z4FlopCounterMode._create_pre_module.<locals>.PopStatec                    s2   ˆj  ˆ ¡ tdd„ |ƒ}t|ƒdkr.|d S |S )Nc                 S   s   t | tjƒr|  ¡ S | S r   r”   rS   r   r   r   rl   8  rm   zNFlopCounterMode._create_pre_module.<locals>.PopState.forward.<locals>.<lambda>r7   r   )rp   rš   r   r^   r—   rŽ   r   r   r™   5  s
    z<FlopCounterMode._create_pre_module.<locals>.PopState.forwardc                    s    ˆj d ˆ kst‚ˆj  ¡  |S )Nr“   )rp   r   r–   r›   rŽ   r   r   rœ   =  s    
z=FlopCounterMode._create_pre_module.<locals>.PopState.backwardNr�   r   rŽ   r   r   ÚPopState4  s   r¥   r¢   )ru   r‚   r¥   r   rŽ   r   r‹   3  s    z"FlopCounterMode._create_pre_moduler   c                 C   s   t | jd  ¡ ƒS )Nrn   )Úsumro   r‡   ©ru   r   r   r   Úget_total_flopsE  s    zFlopCounterMode.get_total_flopsc                 C   s
   t | jƒS )a  Returns the flop counts as a dictionary of dictionaries. The outer
        dictionary is keyed by module name, and the inner dictionary is keyed by
        operation name.

        Returns:
            Dict[str, Dict[Any, int]]: The flop counts as a dictionary.
        )rz   ro   r§   r   r   r   Úget_flop_countsH  s    zFlopCounterMode.get_flop_countsc                    s   |d krˆj }|d krd}dd l}d|_dddg}g }ˆ ¡ ‰ tˆ ƒ‰d‰‡ ‡‡‡fdd	„}ˆj ¡ D ]H}|d
krtqf| d¡d }||krŒqf|||d ƒ}|D ]}	| |	¡ qžqfd
ˆjkröˆsöt	|ƒD ] \}
}	d||
 d  ||
 d< qÆ|d
dƒ| }t
|ƒdk�rd
ddgg}|j||dd�S )Ni?B r   Trr   ZFLOPz% TotalFc                    s¢   t ˆj|   ¡ ƒ}ˆ|ˆ kO ‰d| }g }| ||  t|ˆƒ|ˆ  d d›d�g¡ ˆj|   ¡ D ]:\}}| |d t|ƒ t|ˆƒ|ˆ  d d›d�g¡ qb|S )Nú éd   z.2fú%z - )r¦   ro   r‡   rš   rf   r{   r`   )Úmod_namerh   rM   Úpaddingr‡   r#   Úv©Zglobal_flopsZglobal_suffixZis_global_subsumedru   r   r   Úprocess_mod`  s     ýýz.FlopCounterMode.get_table.<locals>.process_modrn   rw   r7   rª   Ú0z0%)ÚleftÚrightr´   )ÚheadersZcolalign)rh   ÚtabulateZPRESERVE_WHITESPACEr¨   rc   ro   ÚkeysÚcountrš   Ú	enumerater^   )ru   rh   r¶   Úheaderr‡   r±   r€   Z	mod_depthZ
cur_valuesre   Úidxr   r°   r   Ú	get_tableR  s8    
zFlopCounterMode.get_tablec                    s    | j  ¡  |  ¡  tƒ  ¡  | S r   )ro   r‰   r†   ÚsuperÚ	__enter__r§   ©Ú	__class__r   r   r¾   �  s    

zFlopCounterMode.__enter__c                    s.   | j rt|  | j¡ƒ |  ¡  tƒ j|Ž  d S r   )ri   Úprintr¼   rh   rŠ   r½   Ú__exit__)ru   r    r¿   r   r   rÂ   •  s    zFlopCounterMode.__exit__r   c                 C   s~   |r|ni }|||Ž}|j }|| jkrz| j| }tt|||fƒ\}}}|||d|i—Ž}	| jD ]}
| j|
 |  |	7  < q^|S )Nr   )Z_overloadpacketrt   r   r   rp   ro   )ru   ÚfuncÚtypesr    r!   r�   Zfunc_packetZflop_count_funcr   r<   Zparr   r   r   Ú__torch_dispatch__›  s    



z"FlopCounterMode.__torch_dispatch__)Nr   TN)N)r   N)ry   rž   rŸ   Ú__doc__r   r   r   rq   rr   r   rk   Úboolr   r   rv   r†   rŠ   r}   r~   r‘   r‹   r¨   r`   r©   r¼   r¾   rÂ   rÅ   Ú__classcell__r   r   r¿   r   r   Ù   s.       ûû

=c                   @   s   e Zd ZU eed< eed< dS )r   r„   r…   N)ry   rž   rŸ   r   Ú__annotations__r   r   r   r   r   ¨  s   
r   )N)N)N)F)8r   Ztorch.nnrq   Ztorch.utils._pytreer   Útypingr   r   r   r   r   r   Úcollectionsr	   Ztorch.utils._python_dispatchr
   Ztorch.utils.hooksr   Úmathr   Ú__all__ZopsZatenr   rk   r&   r(   r,   r-   rÇ   r1   r6   r9   r=   rN   rO   rP   rQ   ÚmmZaddmmZbmmZbaddbmmZconvolutionZ_convolutionZconvolution_backwardZ'_scaled_dot_product_efficient_attentionZ#_scaled_dot_product_flash_attentionZ0_scaled_dot_product_efficient_attention_backwardZ,_scaled_dot_product_flash_attention_backwardrt   rU   r_   rc   rf   r   r   r   r   r   r   Ú<module>   sx     üûô           õ P