U
    •Ê¹j2$  ã                   @  sà  d Z ddlmZ ddlZddlmZ ddlZddlmZm	Z	m
Z
 ddlmZ ddlmZmZmZ dd	d
ddddgZejejdd�Zedƒe
 d¡ejddœdd„ƒƒƒZedƒejd3ddœdd	„ƒƒZedƒejd4ddœdd
„ƒƒZedƒe
 d¡e
 dd¡ejddœdd„ƒƒƒƒZedƒe
 ddddddddd¡	ejddœd d„ƒƒƒZed!ƒejddœd"d„ƒƒZed#ƒe
 dddddd$d¡ejd5dd'd'd'd(d)d*d(d+œd,d„ƒƒƒZejdd'd'd-œd.d/„ƒZejdd'd'd'd0œd1d2„ƒZ dS )6a&  This file exports ONNX ops for opset 14.

Note [ONNX operators that are added/updated in opset 14]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
New operators:
    HardSwish, Trilu

Updated operators:
    Reshape
    Add, Sub, Mul, Div
    GRU, LSTM, RNN
    BatchNorm, Cumsum, Relu
é    )ÚannotationsN)ÚOptional)Ú
_constantsÚ_type_utilsÚsymbolic_helper)ÚGLOBALS)Ú	_beartypeÚ	jit_utilsÚregistrationÚ	hardswishÚtrilÚtriuÚreshapeÚ
batch_normÚquantized_hardswishÚscaled_dot_product_attentioné   )Zopsetzaten::hardswishÚvzjit_utils.GraphContext)Úgc                 C  s   |   d|¡S )NZ	HardSwish©Úop)r   Úself© r   úZ/var/www/html/bwave/python/venv/lib/python3.8/site-packages/torch/onnx/symbolic_opset14.pyr   (   s    z
aten::trilc                 C  s   | j d||dd�S )NÚTrilur   ©Zupper_ir   ©r   r   ZdiagonalÚoutr   r   r   r   /   s    z
aten::triuc                 C  s   | j d||dd�S )Nr   é   r   r   r   r   r   r   r   5   s    zaten::reshapeTc                 C  s   t j| ||dd�S )Nr   )Z	allowzero)r   Z_reshape_helper)r   r   Úshaper   r   r   r   ;   s    zaten::batch_normÚiÚfc
                 C  sÈ   t  ¡ r8t |||||g¡s8tjdk r8t dddd|¡S t |d¡ t | |||||¡\}}}}| j	d||||||d| |s€dnd|sŠdndd	�
}
|sš|
S |
\}}}| 
| ¡ ¡ | 
| ¡ ¡ |S d S )
Né   ZBatchNormalizationr   zaAll input tensors must have the same `dtype`. Turn off Autocast or export using opset version 15.r   r   r   é   )Z	epsilon_fZ
momentum_fZtraining_mode_iÚoutputs)ÚtorchZis_autocast_enabledr   Zargs_have_same_dtyper   Zexport_onnx_opset_versionZ _onnx_opset_unsupported_detailedZcheck_training_modeZ_batchnorm_helperr   ZsetTypeÚtype)r   ÚinputÚweightZbiasZrunning_meanZrunning_varZtrainingZmomentumÚepsZcudnn_enabledr   ÚresZnew_running_meanZnew_running_varr   r   r   r   E   sT    ÿÿþûú	     ÿ

ö
zquantized::hardswishc                 C  s.   t  | |¡\}}}}t| |ƒ}t  | |||¡S )N)r   Zdequantize_helperr   Zquantize_helper)r   ÚxZop_scaleZop_zero_pointÚ_Úoutputr   r   r   r   }   s    
z"aten::scaled_dot_product_attentionÚbç        Fztorch._C.ValuezOptional[torch._C.Value]ÚfloatÚbool)r   ÚqueryÚkeyÚvalueÚ	attn_maskÚ	dropout_pÚ	is_causalÚscalec              
   C  sÄ  |r|rt  |¡stdƒ‚t  |d¡}t  |¡r:t| |ƒ}|rJt| ||ƒ}t  |¡}tt|ƒƒ}	|	d |	d  |	d< |	d< | j	d||	d�}
|  	d||  	d|¡¡}|  	d|
|  	d|¡¡}|  	d	||¡}t  |¡rÔ|}n¤t
j |¡t
jjk�r<| j	d
t dg¡d�}| j	d
t tdƒ g¡d�}|  	d|||¡}|  	d||¡}n<t
j |¡t
jjk�rb|  	d||¡}ntdt
j |¡› �ƒ‚| j	d|dd�}|dk�r¶|  	d|| j	d
tj|tjd�d�¡}|  	d	||¡S )Nz6is_causal and attn_mask cannot be set at the same timer!   éþÿÿÿéÿÿÿÿZ	Transpose)Zperm_iZMulÚSqrtZMatMulÚConstantr/   ©Zvalue_tÚinfÚWhereÚAddz Unsupported type for attn_mask: ZSoftmax©Zaxis_ir   ZDropout©Zdtype)r   Z_is_noneÚAssertionErrorZ_maybe_get_constÚ_attention_scaleÚ_causal_attention_maskZ_get_tensor_rankÚlistÚranger   r   ÚJitScalarTypeÚ
from_valueZBOOLr%   Útensorr0   ÚFLOATÚ
ValueError)r   r2   r3   r4   r5   r6   r7   r8   Zkey_shape_builtinZkey_transposed_axesZkey_transposedZquery_scaledZkey_transposed_scaledZmul_qkZ
mul_qk_addÚ
const_zeroÚconst_neg_infZattn_weightr   r   r   r   ‹   s^    ÿÿþ


þ

ÿÿ
ÿÿÿ
ý)r   r2   Úreturnc                 C  s    |   d|¡}|   d|| j dtjdgtjd�d�| j dtjtjgtjd�d�¡}| j d|tj |¡ 	¡ d�}| j dtjd	gtj
d�d�}|   d
||   d|¡¡}|S )zºCalculate the scale factor for the attention result.

    Args:
        query: Tensor of shape [..., L, E]

    Returns:
        Scalar scale factor := 1 / math.sqrt(query.size(-1))
    ÚShapeÚSlicer<   r:   rB   r=   ZCast)Zto_iç      ð?ÚDivr;   )r   r%   rJ   Úint64r   Z	INT64_MAXr   rH   rI   Z	onnx_typer0   )r   r2   Úquery_shapeZquery_shape_lastZembedding_sizeÚ	const_oner8   r   r   r   rD   Õ   s$     ÿüýrD   )r   r2   r3   rO   c                 C  s  |   d|¡}|   d|¡}| j dtjdgtjd�d�}| j dtjdgtjd�d�}|   d|||¡}|   d|||¡}| j d||d	d
�}	| j dt dg¡d�}
|   d|
|	¡}| j d|d	d�}| j dt dg¡d�}| j dt tdƒ g¡d�}|   d|   d||¡||¡}|S )až  Create a causal mask for the given query and key tensors.

    Equivalent to::
        mask = torch.ones(L, S, dtype=torch.bool).tril(diagonal=0)
        attn_mask = torch.zeros(L, S, dtype=torch.float)
        attn_mask = attn_mask.masked_fill(not mask, -float('inf'))

    Args:
        query: Tensor of shape [..., L, E]
        key: Tensor of shape [..., S, E]

    Returns:
        Tensor of shape [L, S]
    rP   r<   r:   rB   r=   r9   rQ   ZConcatr   rA   rR   ZExpandr   r   r/   r>   r?   ZEqual)r   r%   rJ   rT   r0   )r   r2   r3   rU   Z	key_shapeZlast_idxZsecond_last_idxZtarget_lengthZsource_lengthÚsizerV   r5   rM   rN   r   r   r   rE   ô   s&       ÿrE   )N)N)Nr/   FN)!Ú__doc__Ú
__future__r   Ú	functoolsÚtypingr   r%   Z
torch.onnxr   r   r   Ztorch.onnx._globalsr   Ztorch.onnx._internalr   r	   r
   Ú__all__ÚpartialZonnx_symbolicZ_onnx_symbolicÚ
parse_argsZbeartyper   r   r   Zquantized_argsr   r   r   r   rD   rE   r   r   r   r   Ú<module>   sd   ù

5    ø$G