U
    yÊ¹jZ  ã                   @   sp  d dl Z ddlmZmZmZmZmZ ddlmZ ddl	m
Z
mZ e je je jgZdd„ Zd	d
„ Zdd„ Zeedddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�edddddœddd�geƒ  dddge
eddœd�eddd„ iƒeejejejejejejejd œd!d"„ƒƒƒZG d#d$„ d$e jjƒZejZdS )%é    Né   )ÚConfigÚautotuneÚcdivÚ
heuristicsÚjit)Úlanguageé   )Úearly_config_pruneÚestimate_matmul_timec                 C   sR   | |kr| S | t kst‚|t ks$t‚t D ]$}| |kr<|  S ||kr(|   S q(d S ©N)Ú_ordered_datatypesÚAssertionError)ÚaÚbÚd© r   úP/var/www/html/bwave/python/venv/lib/python3.8/site-packages/triton/ops/matmul.pyÚget_higher_dtype
   s    r   c                    s   ‡ fdd„S )Nc                    s   | ˆ    ¡ S r   )Zzero_)Únargs©Únamer   r   Ú<lambda>   ó    zinit_to_zero.<locals>.<lambda>r   r   r   r   r   Úinit_to_zero   s    r   c                  C   sŒ   g } dD ]~}dD ]t}dD ]j}dD ]`}|dkr0dnd}|   t|||dd	œ||d
�¡ dD ](}|   t||||d	œ||tdƒd�¡ qVq qqq| S )N)r   é   é   é   é   )é   é    )r    é@   )r    r!   é€   é   r!   r   r   r	   ©ÚBLOCK_MÚBLOCK_NÚBLOCK_KÚSPLIT_K©Ú
num_stagesÚ	num_warps)r   r   é   r   ÚC)r*   r+   Zpre_hook)Úappendr   r   )Úconfigsr*   Zblock_mZblock_kZblock_nr+   Zsplit_kr   r   r   Úget_configs_io_bound   s&     ÿÿ  ÿr0   r"   r#   r    r$   r   r,   r)   r!   r   r   ÚMÚNÚKé
   )r
   Z
perf_modelZtop_k)r/   ÚkeyZprune_configs_byÚEVEN_Kc                 C   s   | d | d | d   dkS )Nr3   r'   r(   r   r   )Úargsr   r   r   r   L   r   r   )Údot_out_dtyper%   r&   r'   ÚGROUP_Mr(   r6   c           (      C   sæ  t  d¡}t  d¡}t  ||¡}t  ||¡}|| }|| }t|||  |ƒ}|| ||  }|| | }|| t  d|¡ }|| t  d|¡ }t  t  || |¡|¡}t  t  || |¡|¡}|| t  d|¡ } | |d d …d f | | d d d …f |   } || d d …d f | |d d d …f |	   }t j||f|d�}!tdt  ||| ¡ƒD ]Ô}"|�rvt  	| ¡}#t  	|¡}$nb||"||   }%t jd|j
jd�}&t j	| | d d d …f |%k |&d�}#t j	|| d d …d f |%k |&d�}$|# |j
j¡}#|$ |j
j¡}$|!t j|#|$|d�7 }!| || | 7 } ||| | 7 }�qV|! |j
j¡}!|| t  d|¡ }|| t  d|¡ }||d d …d f |
 |d d d …f |   }||k d d …d f ||k d d d …f @ }'|dk�rÒt j||!|'d� nt j||!|'d� d S )Nr   r	   )Údtype)r	   r	   )ÚmaskÚother)Z	out_dtype)r;   )ÚtlZ
program_idr   ÚminZarangeZmax_contiguousZmultiple_ofZzerosÚrangeÚloadr:   Z
element_tyÚtoÚdotÚstoreZ
atomic_add)(ÚAÚBr-   r1   r2   r3   Z	stride_amZ	stride_akZ	stride_bkZ	stride_bnZ	stride_cmZ	stride_cnr8   r%   r&   r'   r9   r(   r6   ÚpidZpid_zZgrid_mZgrid_nÚwidthZgroup_idZ
group_sizeZpid_mZpid_nZrmZrnÚramZrbnZrkÚaccÚkr   r   Zk_remainingZ_0r;   r   r   r   Ú_kernel-   sL    +

,,
  ,(
rK   c                   @   s.   e Zd ZeZi Zedd„ ƒZeddd„ƒZdS )Ú_matmulc           	         sª  | j }|  d¡dkr*|  d¡dkr*|  ¡ } | d¡dkrN| d¡dkrN| ¡ }| jd |jd ksjtdƒ‚| j\‰ }|j\}‰| jtjtjtj	fksª|jtjtjtj	fkr²t
j}nt| j|jƒ}t
jˆ ˆf||d�}|d k�r|t
jt
jt
jfkrútj}ntj}nJt|t
jƒ�stdƒ‚|t
jk�r,tj}n |t
jt
jfk�rFtj}ntj}‡ ‡fdd„}t| | ||ˆ ˆ||  d¡|  d¡| d¡| d¡| d¡| d¡|dd	� |S )
Nr   r	   zincompatible dimensions)Údevicer:   z#dot_out_dtype must be a torch.dtypec                    s$   t ˆ | d ƒt ˆ| d ƒ | d fS )Nr%   r&   r(   )r   )ZMETA©r1   r2   r   r   r   ­   r   z_matmul._call.<locals>.<lambda>r,   )r8   r9   )rM   ZstrideÚ
contiguousÚshaper   r:   r=   Zfloat8e4Zfloat8e4b15Zfloat8e5ÚtorchÚfloat16r   ÚemptyÚfloat32Úbfloat16Zint32Ú
isinstancerK   )	r   r   r8   rM   r3   Ú_Zc_dtypeÚcÚgridr   rN   r   Ú_callŒ   sH    

ÿ
   ûz_matmul._callNc                 C   s   t j|||d�S )N)r8   )rL   rZ   )Úctxr   r   r8   r   r   r   Úforward¶   s    z_matmul.forward)N)	Ú__name__Ú
__module__Ú__qualname__rK   ZkernelZ_locksÚstaticmethodrZ   r\   r   r   r   r   rL   ‡   s   
)rL   )rQ   Ú r   r   r   r   r   r   r=   Zmatmul_perf_modelr
   r   rR   rU   rT   r   r   r   r0   Z	constexprrK   ZautogradÚFunctionrL   ÚapplyÚmatmulr   r   r   r   Ú<module>   sd   ìëýè ÿ    ú84