U
    yÊ¹já  ã                   @   s’   d dl Z ddlmZ ddlmZ ddlmZ dd„ Zeejejejdœd	d
„ƒZeejejejdœdd„ƒZ	G dd„ de j
jƒZG dd„ dƒZdS )é    Né   )Újit)Úlanguage)Únext_power_of_2c                 C   s4   | dkrdS | dkrdS | dkr$dS | dkr0dS d	S )
Né€   é   é   é   i   é   i   é   é   © )Únr   r   ú]/var/www/html/bwave/python/venv/lib/python3.8/site-packages/triton/ops/blocksparse/softmax.pyÚ	num_warps   s    r   )ÚROW_SIZEÚ
BLOCK_SIZEÚIS_DENSEc                  C   sô  t  d¡}t  d¡}t  d¡}|t  d¡ | }t  d|
¡| }t  d|
¡| }||| d  }t  |d ¡}t  |d ¡}|| }||| | | 7 }||| | 7 }|rºt  d|
¡}nH|dt  d¡ t  d¡ |  }t j|| | ||k dd�}|| | }||k }t j|| | |tdƒ d�}| t j¡}|}||9 }|d k	�r¦||| 7 }||| 7 }|| d | }|dk||k @ }t j|||  | |dd�}||7 }| t j¡}t  ||k|	@ tdƒ |¡}t  	|¡}t j
| | | ||d� d S )Nr   r   r	   ©ÚmaskÚotherÚinfç        ©r   )ÚtlÚ
program_idÚnum_programsÚarangeÚloadÚfloatÚtoÚfloat32ÚwhereÚsoftmaxÚstore) ÚOutÚAZ	stride_xzÚLUTÚRÚextentÚ	stride_zrÚ	stride_hrÚscaleÚ	is_causalr   r   r   ÚhÚmÚzÚhmÚlane_nÚblock_nÚheaderÚsizeÚoffsetZoff_aÚnsÚoff_lutÚstart_nr   ÚaÚoutÚoff_loÚmask_loÚ
rel_logitsr   r   r   Ú_blocksparse_softmax_fwd   sB    	


 

r?   c           '      C   s"  t  d¡}t  d¡}t  d¡}|t  d¡ | }t  d|¡| }t  d|¡| }||| d  }t  |d ¡}t  |d ¡}|| | | }||| | 7 }||k }|||  | }|||  | }|rÖt  d|¡}nD|dt  d¡ t  d¡ |  }t j|| | |dd�} | | | }t j|| |dd�}!|! t j¡}!t j|| |dd�}"|" t j¡}"t  ||k|@ |!|!k@ d|!¡}!|!|"t  |!|" d¡  }#|d k	�rò|||
 7 }||| 7 }|	| d | }$|$dk|$|	k @ |@ }%t j	|||	  |$ |#|%d� |#| }#| ||  | }&t j	|&| |#|d� d S )Nr   r   r	   r   r   r   )
r   r   r   r   r   r    r!   r"   Úsumr$   )'ZDAZ
stride_zdxZDOutZstride_zdoutr%   Zstride_zoutr,   r'   ZDRr)   r*   r+   Z	stride_err-   r   r   r   r.   r/   r0   r1   r2   r3   r4   r5   r6   Zoff_mnr   ZAsZDOutsr7   r8   r9   r:   ÚdoutÚdar<   r=   ZDAsr   r   r   Ú_blocksparse_softmax_bwdK   sD    


 
rC   c                   @   s0   e Zd Zedd„ ƒZedd„ ƒZedd„ ƒZdS )Ú_softmaxc              	   C   sâ   t jg t j| jd�}| ¡ }t| jd ƒD ]*}t  || |d d …d d …f  d¡f¡}q*|| }t  	|¡}t j
|d d… dd�|dd …< | jdd�d d …df }t j||fdd� d¡}	t  |	|f¡ t j¡ |¡}
|
t| ¡ ƒfS )	N©ÚdtypeÚdevicer   éÿÿÿÿ)Údimr   F)Úas_tupler	   )ÚtorchZtensorZint64rG   ÚcloneÚrangeÚshapeÚcatr@   Z
zeros_likeZcumsumZnonzeroÚstackÚviewÚtypeZint32r    ÚintÚmax)ÚlayoutÚblockrG   Ú_emptyÚsizesr.   Ztotal_sizesÚoffsetsÚcolumnsr4   Úlutr   r   r   Úmake_lutˆ   s    (
z_softmax.make_lutc
                 C   s  |d k	r,t |tjƒr,|jjdks$t‚| ¡ }|jd }
|d |d | |
g}|d krXdn|j}|d krjdn| ¡ }t 	|¡}t
| ||| d¡|||d |d |d |||t|ƒ|	t|ƒd� |  ||¡ || _|| _|| _|| _|| _|| _|j| _|	| _|| _|S )NÚcpur   r   )r   r   r   r   rH   ©r   r   r   r   )Ú
isinstancerK   ZTensorrG   rR   ÚAssertionErrorÚitemrN   ÚstrideÚ
empty_liker?   r   r   Zsave_for_backwardÚspdimsrV   Úmaxlutr,   Ú	rel_shapeÚrel_stridesrF   Ú	rel_dtypeÚis_denser-   )Úctxr:   r,   r>   r-   rd   rV   r[   re   ri   ÚMÚgridrf   rg   r;   r   r   r   Úforward™   sF    

      øz_softmax.forwardc                 C   sô   | j \}}d }| jd r.tj| j| j|jd�}|jd }| jd | jd | j	 |f}t 
|¡}t| || d¡|| d¡|| d¡| j||| jd | jd | jd | jd | j| j	t| jƒ| jt| jƒd� |d d |d d d d d d d d d d d d d d fS )Nr   rE   r   r   rH   r	   r^   )Zsaved_tensorsZneeds_input_gradrK   Zzerosrf   rh   rG   rN   rd   rV   rc   rC   rb   r,   rg   r-   r   re   ri   r   )rj   rA   r;   r[   Zdrrk   rl   rB   r   r   r   Úbackward¿   sT    



       õ
        ûz_softmax.backwardN)Ú__name__Ú
__module__Ú__qualname__Ústaticmethodr\   rm   rn   r   r   r   r   rD   ‡   s   

%rD   c                   @   s(   e Zd Zd	dd„Zddddœdd„ZdS )
r#   Fc                 C   s8   |j | _|| _|| _t | j| j|¡\| _| _|| _d S )N)	rN   rd   rU   rV   rD   r\   r[   re   ri   )ÚselfrU   rV   rG   ri   r   r   r   Ú__init__â   s
    zsoftmax.__init__g      ð?N)r,   r>   r-   c                C   sL   |d k	r$|j |j kr$td|j › �ƒ‚t ||||| j| j| j| j| j¡	}|S )Nz$relative position embedding must be )	rF   Ú
ValueErrorrD   Úapplyrd   rV   r[   re   ri   )rs   r:   r,   r>   r-   r   r   r   Ú__call__é   s           þzsoftmax.__call__)F)ro   rp   rq   rt   rw   r   r   r   r   r#   á   s   
r#   )rK   Ú r   r   r   r   r   Z	constexprr?   rC   ZautogradÚFunctionrD   r#   r   r   r   r   Ú<module>   s    ú6	ö;Z