U
    yÊ¹j…Y  ã                   @  sP  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
 d dlmZ d dlmZmZ ddlmZmZmZmZmZmZmZ ddlmZmZ dd	lmZ dd
lmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z, dd„ Z-dd„ Z.dd„ Z/dd„ Z0dd„ Z1dd„ Z2dd„ Z3e 4¡ dd œd!d"„ƒZ5dTd#ddd$d%œd&d'„Z6d$dd(œd)d*„Z7d+d,„ Z8d-d.„ Z9d#d$d$d$d/d0œd1d2„Z:d$d$d$d3œd4d5„Z;d6d7„ Z<d8d9„ Z=d:Z>d;Z?e>e>e?d<œZ@d=ZAd>ZBeAeAeBd<œZCd?ZDd@dA„ ZEdBdC„ ZFe
dDdEdFgeGƒ eGƒ gdG�ZHdHdI„ ZIdJdK„ ZJdLdM„ ZKdNdO„ ZLdPdQ„ ZMG dRdS„ dSƒZNdS )Ué    )ÚannotationsN)Ú
namedtuple)ÚPath)ÚAnyÚTupleé   )Úadd_external_libsÚcompile_ptx_to_cubinÚget_shared_memory_sizeÚirÚtranslate_llvmir_to_hsacoÚtranslate_llvmir_to_ptxÚtranslate_triton_gpu_to_llvmir)Úget_backendÚpath_to_ptxas)ÚOutOfResources)Úget_cache_manager)Údriver)ÚJITFunctionÚget_cuda_streamÚget_current_deviceÚget_device_capabilityÚversion_key)Úextracté   )Úast_to_ttir)Ú	make_stubc                 C  s*   t  | j¡}| ¡  | ¡  | | ¡ | S ©N)r   Úpass_managerÚcontextÚenable_debugÚadd_inliner_passÚrun)ÚmodÚpm© r%   úW/var/www/html/bwave/python/venv/lib/python3.8/site-packages/triton/compiler/compiler.pyÚinline_triton_ir   s
    
r'   c                 C  s4   t  | j¡}| ¡  t|ƒr&| |¡ | | ¡ | S r   )r   r   r   r    Ú_is_cudaZadd_rewrite_tensor_pointer_passr"   ©r#   Úarchr$   r%   r%   r&   Úttir_compute_capability_rewrite'   s    

r+   c                 C  sl   t | ƒ} t| |ƒ} t | j¡}| ¡  | ¡  | ¡  | ¡  | 	¡  | 
¡  | ¡  | ¡  | | ¡ | S r   )r'   r+   r   r   r   r    r!   Zadd_triton_combine_passZadd_canonicalizer_passZadd_reorder_broadcast_passÚadd_cse_passZadd_licm_passÚadd_symbol_dce_passr"   r)   r%   r%   r&   Úoptimize_ttir2   s    

r.   c                 C  s,   t  | j¡}| ¡  | |¡ | | ¡ | S r   )r   r   r   r    Z$add_convert_triton_to_tritongpu_passr"   )r#   Ú	num_warpsr$   r%   r%   r&   Úttir_to_ttgirB   s
    

r0   c                 C  s˜   t  | j¡}| ¡  | ¡  | ¡  t|tƒr8| |¡ | ¡  | 	¡  | 
|¡ | ¡  | 	¡  | ¡  | ¡  | ¡  | ¡  | ¡  | | ¡ | S r   )r   r   r   r    Zadd_tritongpu_coalesce_passZ,add_tritongpu_remove_layout_conversions_passÚ
isinstanceÚintZ$add_tritongpu_accelerate_matmul_passZ(add_tritongpu_optimize_dot_operands_passZadd_tritongpu_pipeline_passZadd_tritongpu_prefetch_passZ(add_tritongpu_decompose_conversions_passZ'add_tritongpu_reorder_instructions_passr,   r-   r"   )r#   Ú
num_stagesr*   r$   r%   r%   r&   Úoptimize_ttgirJ   s$    



r4   c                 C  sP   |  ¡ D ]&\}}t|ƒdks(t|ƒdkr d S qt| t| ¡ ƒt| ¡ ƒƒ d S )Nr   )ÚitemsÚlenr   ÚlistÚkeysÚvalues)r#   ZlibsÚnameÚpathr%   r%   r&   Ú_add_external_libs_   s    r<   c                 C  s2   |rt | |ƒ t|ƒr"t| |dƒS t| ddƒS d S )NFr   T)r<   r(   r   )r#   Úextern_libsr*   r%   r%   r&   Úttgir_to_llirf   s
    
r>   r2   )Úreturnc                 C  s^   t | tƒst‚tt|  d¡ƒ\}}|dkr2d| S |dkrBd| S |dkrRd| S tdƒ‚d	S )
zK
    Get the highest PTX version supported by the current CUDA driver.
    Ú.é   éP   é   éF   é
   é?   z'Triton only support CUDA 10.0 or higherN)r1   ÚstrÚAssertionErrorÚmapr2   ÚsplitÚRuntimeError)Úcuda_versionÚmajorÚminorr%   r%   r&   Úptx_get_versionr   s    rO   r   rG   )r#   r*   Úptx_versionr?   c                 C  s&   |dkrt ƒ \}}t|ƒ}t| ||ƒS )zr
    Translate TritonGPU module to PTX code.
    :param mod: a TritonGPU dialect module
    :return: PTX code
    N)r   rO   r   )r#   r*   rP   Ú_rL   r%   r%   r&   Úllir_to_ptx‚   s    
rR   )Úptxr*   c                 C  s   t ƒ \}}t| ||ƒS )zˆ
    Compile TritonGPU module to cubin.
    :param ptx: ptx code
    :param compute_capability: compute capability
    :return: str
    )r   r	   )rS   r*   ZptxasrQ   r%   r%   r&   Úptx_to_cubinŽ   s    
rT   c                 C  s¼   ddddddddg}| d	 }t  d
|¡ d	¡ ¡ }d| d }tj ttƒj	 
¡ d¡}i }d	}|D ]0}|| }	tj |	¡rb|	|dt|ƒ < |d	7 }qb|| }
tj |
¡r¸|
|dt|ƒ < |S )Nz	opencl.bczocml.bczockl.bczoclc_finite_only_off.bczoclc_daz_opt_off.bcz!oclc_correctly_rounded_sqrt_on.bczoclc_unsafe_math_off.bczoclc_wavefrontsize64_on.bcr   zgfx(\w+)Zoclc_isa_version_z.bczthird_party/rocm/lib/bitcode/Zlibrary_)ÚreÚsearchÚgroupÚstripÚosr;   Újoinr   Ú__file__ÚparentÚresolveÚexistsrG   )r*   Z#gpu_arch_agnostic_bitcode_librariesÚgfx_archZgfx_arch_idZ!gpu_arch_specific_bitcode_libraryZbitcode_path_dirZamdgcn_bitcode_pathsÚiZbc_libZbc_pathZbc_gfx_pathr%   r%   r&   Úget_amdgcn_bitcode_paths›   s0    ù	
ra   c                  C  sÆ   zªt jddd�} t | d ¡ ¡ }t d|¡ d¡ ¡  	d¡}|d }|d  	d	¡}|d }d
}t
|ƒdkr dt d|d ¡ d¡ d t d|d ¡ d¡ }|||gW S  tk
rÀ   Y dS X dS )z•
    get the amdgpu fulll ISA details for compiling:
    i.e., arch_triple: amdgcn-amd-amdhsa; arch_name: gfx906; arch_features: sramecc+:xnack-
    Z	ROCM_PATHz	/opt/rocm)Údefaultz/bin/rocminfozamd.*r   z--r   ú:Ú é   ú+z\w+z,-r   N)rY   ÚgetenvÚ
subprocessÚcheck_outputÚdecoderU   rV   rW   rX   rJ   r6   ÚBaseException)Zrocm_path_dirZrocminfoZgfx_arch_detailsZarch_tripleZarch_name_featuresZ	arch_nameZarch_featuresr%   r%   r&   Úget_amdgpu_arch_fulldetails¹   s    ÿrl   zTuple[str, str])r#   r_   Ú
gfx_tripleÚgfx_featuresr?   c                 C  s   t | |||ƒS )zÊ
    Translate TritonGPU module to HSACO code based on full details of gpu architecture.
    :param mod: a TritonGPU dialect module
    :return:
        - AMDGCN code
        - Path to HSACO object
    )r   )r#   r_   rm   rn   r%   r%   r&   Úllir_to_amdgcn_and_hsacoÐ   s    ro   )ÚsrcÚpatternr?   c                 C  s>   | st ‚|  d¡D ]&}| ¡ }| |¡r| ¡ d   S qdS )zd
    Get kernel name from PTX code.
    This Kernel name is required when launching the kernel.
    Ú
éÿÿÿÿN)rH   rJ   rX   Ú
startswith)rp   rq   Úliner%   r%   r&   Úget_kernel_nameÞ   s
    
rv   c                 C  s*   t  d| ¡}|d k	r&dt| d¡ƒ S | S )Nz!tt\.ptr<(.*)>Ú*r   )rU   rV   Úconvert_type_reprrW   )ÚxÚmatchr%   r%   r&   rx   ë   s    rx   c                   sè   t | tƒr¸|d }|d }| dtƒ ¡}| dd¡}| dd¡}| dd	¡}d
d„ ‰ ‡ fdd„|D ƒ}	| j› dd | ¡ ¡› d|	› d|› d|› d|› d|› d|› �}
t |
 	d¡¡ 
¡ S t | tƒsÆt‚t t| ƒ ¡ tƒ   	d¡¡ 
¡ S )NÚconfigsÚ	signatureÚ	constantsr/   é   r3   re   ÚdebugFc                 S  s   t | jƒt | jƒfS r   )ÚsortedÚdivisible_by_16Ú
equal_to_1)Úconfr%   r%   r&   Ú<lambda>û   ó    zmake_hash.<locals>.<lambda>c                   s   g | ]}ˆ |ƒ‘qS r%   r%   )Ú.0rƒ   ©Zget_conf_keyr%   r&   Ú
<listcomp>ü   s     zmake_hash.<locals>.<listcomp>ú-rd   zutf-8)r1   r   ÚgetÚdictÚ	cache_keyrZ   r9   ÚhashlibÚmd5ÚencodeÚ	hexdigestrG   rH   r   Ú	read_textr   )Úfnr*   Úkwargsr{   r|   r}   r/   r3   r   Zconfigs_keyÚkeyr%   r‡   r&   Ú	make_hashò   s    
>r•   z`^\s*tt\.func\s+(?:public\s+)?(@\w+)(\((?:%\w+: \S+(?: \{\S+ = \S+ : \S+\})?(?:, )?)*\))\s*\{\s*$z=\.(?:visible|extern)\s+\.(?:entry|func)\s+(\w+)\s*\(([^)]*)\))ÚttirÚttgirrS   z-%\w+: ([^,^\)\s]+)(?: \{\S+ = \S+ : \S+\})?,?z\.param\s+\.(\w+)z&"triton_gpu.num-warps"\s?=\s?(\d+)\s?:c                 C  s2   dd„ }i }| D ]}|| | ƒr| | ||< q|S )Nc              	   S  s0   zt  | ¡ W dS  ttfk
r*   Y dS X d S )NTF)ÚjsonÚdumpsÚ	TypeErrorÚOverflowError)ry   r%   r%   r&   Ú_is_jsonable  s
    
z-_get_jsonable_constants.<locals>._is_jsonabler%   )r}   rœ   Zserialized_constantsZconstantr%   r%   r&   Ú_get_jsonable_constants  s    r�   c                 C  s   t  | |¡}||_|S r   )r   Úparse_mlir_moduler   )r;   r   Úmoduler%   r%   r&   rž   +  s    rž   Úinstance_descriptorr�   r‚   )Údefaultsc                 C  s
   t | tƒS r   )r1   r2   ©r*   r%   r%   r&   r(   6  s    r(   c                 C  sl   zdd l }W n tk
r(   tdƒ‚Y nX | d krh|jjd krbtƒ }t|ƒ} | d d | d  } ntƒ } | S )Nr   z'Triton requires PyTorch to be installedrE   r   )ÚtorchÚImportErrorÚversionÚhipr   r   rl   )Z
capabilityr£   Údevicer%   r%   r&   Úget_architecture_descriptor:  s    r¨   c                   s‚   |  t| ƒ¡ t|ƒD ]&}|| dks2|| d kr| |¡ q| ‰tj dˆd ¡‰ ˆ d krdtdƒ‚dd„ ‡ ‡fdd„f|d< d S )	Nrd   ZMI_GPU_ARCHr   z gfx_arch is None (not specified)c                 S  s   t | ƒ ¡ S r   ©r   r‘   ©r;   r%   r%   r&   r„   T  r…   z!add_rocm_stages.<locals>.<lambda>c                   s   t | ˆ ˆd ˆd ƒS )Nr   r   )ro   ©rp   ©r_   Zgfx_arch_full_detailsr%   r&   r„   U  s   þÚamdgcn)Úupdatera   r7   ÚpoprY   ÚenvironrŠ   rK   )r*   r=   Ústagesr”   r%   r¬   r&   Úadd_rocm_stagesI  s    ÿr²   c                   s4   dd„ ‡ fdd„f|d< dd„ ‡ fdd„f|d< d S )Nc                 S  s   t | ƒ ¡ S r   r©   rª   r%   r%   r&   r„   \  r…   z!add_cuda_stages.<locals>.<lambda>c                   s
   t | ˆ ƒS r   )rR   r«   r¢   r%   r&   r„   ]  r…   rS   c                 S  s   t | ƒ ¡ S r   )r   Ú
read_bytesrª   r%   r%   r&   r„   ^  r…   c                   s
   t | ˆ ƒS r   )rT   r«   r¢   r%   r&   r„   _  r…   Úcubinr%   )r*   r=   r±   r%   r¢   r&   Úadd_cuda_stagesZ  s    
ÿ
ÿrµ   c           "   	     s*  |  dd¡}t|ƒ}|dkr.t|  dd ¡ƒ‰ nt|ƒ}|s>t‚|jf |Ž‰ |dkoXtˆ ƒ}|dkof| }t ¡ ‰|  dtƒ ¡‰|  dd¡‰|  d|r ˆ d	kr d
nd¡‰|  dtƒ ¡‰ˆd krÂtƒ ‰|  dd¡‰tƒ }‡fdd„d f|d< ‡fdd„‡ ‡‡‡‡	fdd„f|d< ‡fdd„‡ ‡‡fdd„f|d< dd„ ‡ ‡fdd„f|d< |�rZtˆ ˆ|ƒ n"|�rnt	ˆ ˆ|ƒ n| 
ˆ ˆ|¡ tˆtƒ�r|  dd ¡‰|d ‰	ˆd k�r®tƒ g‰tˆƒdk�sÀt‚ˆ|d< ˆj}d}tˆ	tƒ�rödd „ tˆ	 d!¡ƒD ƒ‰	ˆ	|d< �ntˆtƒ�st‚tj ˆ¡ d"¡\}	}
tˆƒ ¡ }dd l}| t|
 ||j¡}| d¡| d¡ }‰	| t|
 ˆ	¡}|
dk�rÒ| t|¡}t|ƒdk�s¢td#ƒ‚d|k�sÆt |d ƒˆk�sÆtd$ƒ‚t |d ƒ‰d%d&„ |D ƒ}d'd „ t|ƒD ƒ‰	t!| "¡ ƒ #|
¡}|�s|�rt$|ˆ	ˆƒ}n| %|ˆ	ˆ¡}t&t'ˆˆ f|Žƒ}tˆtƒ�rXˆjd }}ntj ˆ¡ d"¡\}}d }|› d(�}| (|¡�pŠi }|  |¡}|d k	�rÀt)|ƒ�}t* +|¡}W 5 Q R X n<ˆˆt,ˆƒˆˆ d)œ}|d*k�rüd+|k�sðtd,ƒ‚|d+ |d+< ||d< t!| "¡ ƒ #|¡}tƒ }ˆ}t!| -¡ ƒ|d … D �]´\}
\}}|› d"|
› �}|
|k�rd|ˆƒ}nÂ|  |¡}|d k�rÜ||ƒ}td-k�r¾|› d.�} | .|d |¡||< | .|d | ¡|| < n| .||¡||< | .||¡ nJ|
d-k�r|› d.�} |  | ¡}!|!d k	�std/ƒ‚||ƒ||!ƒf}n||ƒ}|
d0k�r:|||
< n(|
d-k�rVt|d ƒ||
< nt|ƒ||
< |
dk�r‚d+|k�r‚t/|ƒ|d+< |
d*k�rœt0|d1d2�|d3< |
d-k�rÆt0|d d4d2�|d3< |d |d5< |�sä|�sä| 1|
||||¡ |}�q4|d k�r|j.t* 2|¡|dd6�||< | 3||¡ t4ˆ|||ƒS )7NÚdevice_typeÚcuda©r·   r¦   Úccr}   r/   r~   r3   éK   re   r   r=   r   Fc                   s   ˆ S r   r%   rª   )r’   r%   r&   r„   {  r…   zcompile.<locals>.<lambda>Úastc                   s
   t | ˆ ƒS r   ©rž   rª   ©r   r%   r&   r„   |  r…   c              	     s   t t| ˆˆd ˆˆˆ d�ˆ ƒS )Nr   )r   r*   )r.   r   r«   )r*   r{   r}   r   r|   r%   r&   r„   }  r…   r–   c                   s
   t | ˆ ƒS r   r¼   rª   r½   r%   r&   r„   ~  r…   c                   s   t t| ˆƒˆˆ ƒS r   )r4   r0   r«   )r*   r3   r/   r%   r&   r„     r…   r—   c                 S  s   t | ƒ ¡ S r   r©   rª   r%   r%   r&   r„   €  r…   c                   s   t | ˆˆ ƒS r   )r>   r«   )r*   r=   r%   r&   r„   �  r…   Zllirr{   r|   r   r   c                 S  s   i | ]\}}||  ¡ “qS r%   )rX   ©r†   ÚkÚvr%   r%   r&   Ú
<dictcomp>”  s      zcompile.<locals>.<dictcomp>ú,r@   z(Expected exactly one match for num_warpsz6num_warps in ttgir does not match num_warps in compilec                 S  s   g | ]}t |ƒ‘qS r%   )rx   )r†   Útyr%   r%   r&   rˆ   £  s     zcompile.<locals>.<listcomp>c                 S  s   i | ]\}}||“qS r%   r%   r¾   r%   r%   r&   rÁ   ¤  s      z.json)r/   r3   r}   r   r*   rS   Úsharedz/ptx compilation must provide shared memory sizer­   z.hsaco_pathz?Expected to have hsaco_path in metadata when we have the amdgcnr´   z	// .globl)rq   r:   z.globlÚ
hsaco_path)Úbinary)5rŠ   r   r¨   rH   r(   r   r   r‹   rµ   r²   Z
add_stagesr1   r   r    r6   Ú__name__rG   Ú	enumeraterJ   rY   r;   Úbasenamer   r‘   rU   rV   Úprototype_patternÚ	MULTILINErW   ÚfindallÚarg_type_patternÚttgir_num_warps_patternr2   r7   r8   Úindexr   Zmake_launcher_stubr   r•   Z	get_groupÚopenr˜   Úloadr�   r5   Úputr
   rv   Zadd_meta_infor™   Z	put_groupÚCompiledKernel)"r’   r“   r¶   Z_device_backendZis_cudaZis_hipr±   r:   Zfirst_stagerQ   Zir_namerp   rU   rz   ÚtypesZnum_warps_matchesZ	param_tysÚso_pathZfn_cache_managerÚextÚmetadataZmetadata_filenameZmetadata_groupÚmetadata_pathÚfÚasmrŸ   ÚparseZcompile_kernelZir_filenameZnext_moduler;   Zextra_file_nameZ
hasco_pathr%   )
r*   r{   r}   r   r   r=   r’   r3   r/   r|   r&   Úcompileb  s   
ÿ
ÿÿ

$
ÿþ


ü
"














rÜ   c                      sF   e Zd ZdZdZdd„ Zdd„ Z‡ fdd„Zdd	„ Zdd
d„Z	‡  Z
S )rÓ   Nc                 C  s´   dd l }|j d|¡}|j |¡}|| _|j |¡ t|dƒ| _d|krP|d nd| _	|d | _
|d | _|d | _|d | _| jd	kr’t| jƒnd | _|| _|| _d | _d | _d S )
Nr   Z__triton_launcherZlaunchrÄ   r/   r3   r}   r¶   r¸   )Zimportlib.utilÚutilÚspec_from_file_locationÚmodule_from_specr’   ÚloaderÚexec_moduleÚgetattrÚ	c_wrapperrÄ   r/   r3   r}   r¶   r   Údevice_backendrÚ   r×   Ú	cu_moduleÚcu_function)Úselfr’   rÕ   r×   rÚ   Ú	importlibÚspecr#   r%   r%   r&   Ú__init__  s     



zCompiledKernel.__init__c           	      C  sÞ   | j d k	rd S | jdkrNtƒ }tjdtjditj }tj |¡d }tjj	}n8| j
sXt‚| j
 ¡ }| j
 ¡ }| j
 |¡d }| j
 ¡ }| j|kržt| j|dƒ‚|| jd | j| | j|ƒ\}}}}|| _|| _|| _ || _d S )Nr¸   rÅ   r´   Zmax_shared_memzshared memoryr:   )rå   r¶   r   r   ZHIPÚCUDAÚbackendÚutilsZget_device_propertiesZload_binaryrä   rH   Zget_kernel_binZget_load_binary_fnrÄ   r   r×   rÚ   Ún_spillsÚn_regsræ   )	rç   r§   Zbin_pathZ
max_sharedZfn_load_binaryr#   Úfuncrï   rî   r%   r%   r&   Ú_init_handles$  s2    

  þý





$zCompiledKernel._init_handlesc                   s   |dkr|   ¡  tƒ  |¡S )Nrã   )rñ   ÚsuperÚ__getattribute__)rç   r:   ©Ú	__class__r%   r&   ró   A  s    zCompiledKernel.__getattribute__c                   s    ˆ  ¡  d dœ‡ ‡fdd„
}|S )N)Ústreamc                   sf   | d kr*ˆj dkrtƒ } ntˆj ƒ d ¡} ˆjˆ d ˆ d ˆ d ˆjˆj| ˆjtj	tj
ˆf
|žŽ  d S )N)r·   Zrocmr   r   r   )r¶   r   r   Ú
get_streamrã   r/   rÄ   ræ   rÓ   Úlaunch_enter_hookÚlaunch_exit_hook)rö   Úargs©Úgridrç   r%   r&   ÚrunnerI  s    
$  ÿÿz*CompiledKernel.__getitem__.<locals>.runner)rñ   )rç   rü   rý   r%   rû   r&   Ú__getitem__F  s    zCompiledKernel.__getitem__c              	   C  sv   d| j kr| j d S t ¡ \}}z6t|dƒ�}| | j d ¡ W 5 Q R X t||ƒ| _W 5 t |¡ X | j| j d< | jS )NÚsassÚwbr´   )	rÚ   ÚtempfileÚmkstemprY   ÚremoverÐ   Úwriter   rÿ   )rç   ZfunÚfdr;   r´   r%   r%   r&   Úget_sassS  s    

zCompiledKernel.get_sass)N)rÇ   Ú
__module__Ú__qualname__rø   rù   rê   rñ   ró   rþ   r  Ú__classcell__r%   r%   rô   r&   rÓ     s   rÓ   )N)OÚ
__future__r   Ú	functoolsr�   r˜   rY   rU   rh   r  Úcollectionsr   Úpathlibr   Útypingr   r   Z_C.libtriton.tritonr   r	   r
   r   r   r   r   Zcommon.backendr   r   Zruntime.autotunerr   Zruntime.cacher   Zruntime.driverr   Zruntime.jitr   r   r   r   r   Ztools.disasmr   Zcode_generatorr   Zmake_launcherr   r'   r+   r.   r0   r4   r<   r>   Ú	lru_cacherO   rR   rT   ra   rl   ro   rv   rx   r•   Zmlir_prototype_patternZptx_prototype_patternrÊ   Zmlir_arg_type_patternZptx_arg_type_patternrÍ   rÎ   r�   rž   Úsetr    r(   r¨   r²   rµ   rÜ   rÓ   r%   r%   r%   r&   Ú<module>   sv   $ýý %