U yʹj? @s*ddlmZddlZddlmZddlmZmZmZm Z m Z ddl m Z ddl mZe d ZGd d d eZd d ddddZd d ddddZddddddZdddddddZdddddddZdddd d"d#d$d%Zddd dd&d'd(Zddd dd&d)d*Zddd dd&d+d,Zddd dd&d-d.Zddd dd&d/d0Zdddd dd1d2d3Zddd dd&d4d5Zddd d"d&d6d7Z ddd dd&d8d9Z!ddd dd&d:d;Z"ddd dd&dd?Z$ddd dd&d@dAZ%dd dBdCdDZ&ddd dd&dEdFZ'ddd dd&dGdHZ(ddd dd&dIdJZ)dddKdLdMZ*dd ddNdOdPZ+ddddNdQdRZ,ddSdTdUdVZ-ddd dd&dWdXZ.ddd dd&dYdZZ/ddd dd&d[d\Z0ddd dd&d]d^Z1ddd dd&d_d`Z2ddd dd&dadbZ3d d d ddcdddeZ4dfdd ddgdhdiZ5ddfd ddjdkdlZ6ddfd ddjdmdnZ7dd d ddodpdqZ8dddd ddrdsdtZ9dd ddNdudvZ:ddfd ddwdxdyZ;ddd dd#dzd{Zddd dd|ddZ?ddZ@ddZAddZBddZCddZDddZEddZFddZGdddddddd dd ddZHddZIddZJdddddd ddddZKddddd ddddZLddddd ddddZMddddd ddddZNddddd ddddZOddddd ddddZPddddd ddddZQddddd ddddZRddddd ddddZSddddd ddddZTddddd ddddZUdddd ddddZVdd d ddddZWdd d ddddZXdd dÜddńZYddd ddƜddȄZZeYddgdˍdd dd̜dd΄Z[eYddgdˍdd dd̜ddЄZ\eYddgdˍdd dd̜dd҄Z]eYddgdˍdd dd̜ddԄZ^eYddgdˍdd dd̜ddքZ_eYddgdˍdd dd̜dd؄Z`dd dd̜ddڄZaddfddۜdd݄Zbddfddۜdd߄ZcddfddۜddZdd ddddZeddd ddddZfdddd d ddddZgddZhdddZidd ddddZjdd ddddZkdS)) annotationsNwraps)ListOptionalSequenceTupleTypeVar)ir)coreTcseZdZfddZZS)IncompatibleTypeErrorImplcs@||_||_d|jd|j|_tt||jdS)Nzinvalid operands of type  and )type_atype_b__repr__messagesuperr__init__)selfrr __class__W/var/www/html/bwave/python/venv/lib/python3.8/site-packages/triton/language/semantic.pyrsz"IncompatibleTypeErrorImpl.__init__)__name__ __module__ __qualname__r __classcell__rrrrrsrintz ir.builderz tl.tensor)axisbuilderreturncCst||tjSN)tltensorZcreate_get_program_idint32r!r"rrr program_idsr)cCst||tjSr$)r%r&Zcreate_get_num_programsr'r(rrr num_programssr*ztl.dtype)a_tyb_tyr#cCsx|j}|j}|j}|j}||kr0||kr,|S|S|tjjjkrN||krJ|S|S|tjjjkrl||krh|S|SdsttdSNF) int_bitwidthint_signednessr%dtypeZ SIGNEDNESSZUNSIGNEDAssertionError)r+r,Za_rankZb_rankZa_snZb_snrrrinteger_promote_impl'sr2bool)r+r, div_or_modr#cCs|s|rtjS|s&|r,tjS|s<|rL|rFtjStjS|s\|r|rftjS|r||r|tjStjS| r| sdst |r|j |j krt d| d| dt||S)NFzCannot use /, #, or % with rx because they have different signedness;this is unlikely to result in a useful answer. Cast them to the same signedness.)Zis_fp64r%Zfloat64is_fp32float32is_fp16float16is_bf16bfloat16is_intr1r/ ValueErrorrr2)r+r,r4rrrcomputation_type_impl7s& r>None)rr allow_ptr_ar#cCsF|rB|st|||r0||kr0t|||rBt||dSr$)is_ptrr is_floating)rrr@rrrcheck_ptr_type_impl]s  rCFTzTuple[tl.tensor, tl.tensor])lhsrhsr"r#c Csxt|||\}}|jj}|jj}t|||t||||rp|sp|spt|||} t|| |}t|| |}||fSr$)broadcast_impl_valuetypescalarrCrAr>cast) rDrEr"Z allow_lhs_ptrZ allow_rhs_ptrZarithmetic_checkr4Z lhs_sca_tyZ rhs_sca_ty ret_sca_tyrrrbinary_op_type_checking_implis     rK)inputotherr"r#cCst|||dd\}}|jj}|jj}|r>|s>||}}|r`t||j|j|jS|rt| |j|j|jS| rt| |j|j|jSdst dSNTF) rKrGrHrAr%r& create_addptrhandlerBZ create_faddr<Z create_addr1rLrMr"input_scalar_tyother_scalar_tyrrradd}s rTcCst|||dd\}}|jj}|rDt||jt||j|jS| rft| |j|j|jS| rt| |j|j|jSdst dSrN)rKrGrHrAr%r&rOrPminusrBZ create_fsubr<Z create_subr1rLrMr" scalar_tyrrrsubsrXcCsht|||\}}|jj}|r:t||j|j|jS|r\t| |j|j|jSdsdt dSr-) rKrGrHrBr%r&Z create_fmulrPr<Z create_mulr1rVrrrmulsrYcCst|||dddd\}}|jj}|jj}|rF|rFt|||}n|rd|rdt|||}nn|r|rt|tj|}t|tj|}n@|r|r|j|jkrt|||}qt|||}ndst t | |j |j |jSNFT) rKrGrHrBr<rIr%r7Zfp_mantissa_widthr1r& create_fdivrPrQrrrtruedivs  r\cCst|||dddd\}}|jj}|jj}|r|rt||}t|||}t|||}|r|t| |j |j |jSt| |j |j |jSdst dSrZ) rKrGrHr<r2rI is_int_signedr%r&Z create_sdivrPZ create_udivr1)rLrMr"rRrSret_tyrrrfloordivs   r_)rLrM ieee_roundingr"r#cCs^|jj}|jj}|r |s(tdt|||dddd\}}||j|j}t||jS)Nz4both operands of fdiv must have floating scalar typeFT) rGrHrBr=rKr[rPr%r&)rLrMr`r"rRrSretrrrfdivsrbc Cst|||dddd\}}|jj}|jj}|rXt|ttt||d|||||}|S|r|j |j krt d| d| d| rt ||j|j|jSt ||j|j|jSdstdS)NFTz Cannot mod z by r5)rKrGrHrBrXrYfloorrbr<r/r=rr]r%r&Z create_sremrPZ create_uremr1)rLrMr"rWrSrarrrmods$  rdcCszt|||ddd\}}|jj}|jj}|r6|s@t||t||}||kr^t|||}||krrt|||}||fSr-)rKrGrHr<rr2rI)rLrMr" input_sca_tyZ other_sca_tyrJrrrbitwise_op_type_checking_impl s    rfcCs*t|||\}}t||j|j|jSr$)rfr%r&Z create_andrPrGrLrMr"rrrand_srhcCs*t|||\}}t||j|j|jSr$)rfr%r&Z create_orrPrGrgrrror_"sricCs*t|||\}}t||j|j|jSr$)rfr%r&Z create_xorrPrGrgrrrxor_)srjcCsD|jst|td|}|js8t|td|}t|||SNint1)rGis_int1bitcastr%r0rhrgrrr logical_and0s   rocCsD|jst|td|}|js8t|td|}t|||Srk)rGrmrnr%r0rirgrrr logical_or8s   rprLr"cCs&|jst|td|}t||Srk)rGrmrnr%r0invertrqrrrnot_@s rscCs*t|||\}}t||j|j|jSr$)rfr%r&Z create_lshrrPrGrgrrrlshrFsrtcCs*t|||\}}t||j|j|jSr$)rfr%r&Z create_ashrrPrGrgrrrashrMsrucCs*t|||\}}t||j|j|jSr$)rfr%r&Z create_shlrPrGrgrrrshlTsrv)rLr#cCs|Sr$r)rLrrrplus_srw)rLr"r#cCsH|jj}|r$td|dt||||}t |||S)Nz$wrong type argument to unary minus ()) rGrHrAr=rr%r&get_null_valueto_irrX)rLr"re_0rrrrUcs rUcCsP|jj}|s|r,td|dt|| ||}t |||S)Nz%wrong type argument to unary invert (rx) rGrHrArBr=rr%r&Zget_all_ones_valuerzrj)rLr"reZ_1rrrrrls rrz tl.block_type)vr#cCs&|jstjS|jj}ttj|Sr$)rGis_blockr%rlshape block_type)r|r~rrr _bool_likexs rcCst|||\}}|jj}|rt|jj}||dt|jj|}t||j ||S)Nr ) listrGr~insertr%rrHr&create_expand_dimsrP)rLr!r"rr^rrr expand_dims s  r)rDrE can_reorderr"r#cCsX|s tdt|jdkstt|jj|jd|jdg}t||j |j |S)Nz;current implementation of `cat` always may reorder elementsr r) r1lenr~r%rrGrHr&Z create_catrP)rDrErr"ret_typerrrcat's "rcCsJt|jdkrtdt|jj|jd|jdg}t||j |S)Nr z!Only 2D tensors can be transposedr r) rr~r=r%rrGrHr&Z create_transrP)rLr"rrrrtrans.s r)rLr~r"r#c Cs|js.t|j|}t||j||S|j}t|t|kr\t d|d|||krh|St |D]F\}}|||krp|dkrpt d||d|d|d|d| qpt|jj |}t| |j||S)Nz!Cannot broadcast, rank mismatch: z, r z3Cannot broadcast, the expanded size of the tensor (z ) must match the existing size (z) at non-singleton dimension : ) rGr}r%rr&rrPget_block_shapesrr= enumeraterHcreate_broadcast)rLr~r"r^Z src_shapeiitemrrrbroadcast_impl_shape5s  ,rc Csp|j}|j}|rJ|sJt|j|j}t||j| |}n|s|rt|j|j}t||j| |}n|rh|rh| }| }t |t |kr t t |t |D]8}t| |jdt|jdg|}|j}| }qn`t |t |krlt t |t |D]:}t| |jdt|jdg|}|j}| }q0t |t |kst g}t|D]|\} } || } | dkr|| nT| dkr|| n>| | kr|| n(tdt| dt| dt| q||kr:t|j|} t||j|| }||krht|j|} t||j|| }||fS)Nrr z?Cannot make_shape_compatible: incompatible dimensions at index rr)rGr}r%rrHr~r&rrPrrrangerr1rappendr=strr) rDrEr"Zlhs_tyZrhs_tyZ lhs_shapeZ rhs_shapedim ret_shaperleftrightr^rrrrFIsb& &          rF)rLdst_tyr"r#cCs|j}|r"t|j|j}||kr.|S|j}|j}|sJ|rVt|||S|j}|j}||krt dt |dt |t | |j |||S)Nz!Cannot bitcast data-type of size z to data-type of size )rGr}r%rrHrrArIprimitive_bitwidthr=rr&create_bitcastrPrz)rLrr"src_ty src_sca_ty dst_sca_tyZsrc_bitsZdst_bitsrrrrns$ rncCs t|tSr$rr )archrrr_is_cudasrc Cs|j}t|tjr|j}|r4t|j|j}||kr@|S|j}|j}t |j r||j dkr|| sp| r|t dt|r|s|r|rt||j|||S|r|r|r|stt|tj|||S|o |o |j|jk}|r0t||j|||S|oN|oN|j|jk}|rrt||j|||S|r |r |j|jks|j|jkr | o|! }|!r|j"|} t|#| |j"} t$|| |St|%|j||||S|&r|r|!rT|j"|} t|#| |j"} t$|| |S| rzt|'|j|||St|(|j|||S|r|&r|!s| st|)|j|||St|*|j|||S|+rf|rf|j} | dkr6t|,|j|||S| dkrft$t|tj-|t|.dtj-|S|r|+rt|/|j|||S|+r|+rt|0|j|||Sdst1d|d|dS) NYzUStandard tl.float8e4 format will be deprecated on SM < 89. Please use tl.float8e4b15.@r rFz cannot cast z to )2rGrr% constexprrr}rrHrrrZis_fp8e4warningswarnDeprecationWarningZis_fp8rBr&Zcreate_fp_to_fprPrzr8r6r:rIr7rZcreate_fp_truncZ create_fp_extr<r.r/r]is_boolr0ryrcreate_int_castZis_standard_floatingZcreate_fp_to_siZcreate_fp_to_uiZcreate_ui_to_fpZcreate_si_to_fprAZcreate_ptr_to_intint64 get_int64Zcreate_int_to_ptrrr1) rLrr"rrrZ truncate_fpZext_fpZ sign_extendtyr{ZbitwidthrrrrIs                       rIcCsDtjj}|r@|dkrtjj}n"|dkr0tjj}ntd|d|S)Nz.ca.cgCache modifier  not supported)r CACHE_MODIFIERNONECACGr=cache_modifiercacherrr_str_to_load_cache_modifiers  rcCshtjj}|rd|dkrtjj}nF|dkr0tjj}n4|dkrBtjj}n"|dkrTtjj}ntd|d|S)Nz.wbrz.csz.wtrr)r rrZWBrCSZWTr=rrrr_str_to_store_cache_modifier!s    rcCsDtjj}|r@|dkrtjj}n"|dkr0tjj}ntd|d|S)NZ evict_lastZ evict_firstzEviction policy r)r ZEVICTION_POLICYZNORMALZ EVICT_LASTZ EVICT_FIRSTr=)eviction_policyevictionrrr_str_to_eviction_policy1s  rcCs@d}|r<|dkrtjj}n"|dkr,tjj}ntd|d|S)NzeronanzPadding option r)r PADDING_OPTIONZPAD_ZEROPAD_NANr=)padding_optionpaddingrrr_str_to_padding_option=s  rcCshtjj}|rd|dkrtjj}nF|dkr0tjj}n4|dkrBtjj}n"|dkrTtjj}ntd|d|S)NacquirereleaseZacq_relrelaxedzMemory semantic r)r Z MEM_SEMANTICZACQUIRE_RELEASEZACQUIREZRELEASEZRELAXEDr=)Z sem_optionsemrrr _str_to_semIs    rcCs|rt|ds|g}dd|D}|D],}t|trNd|krLt|ks&ntq&t|dksdtt|tt|kstdt|StS)N__iter__cSs"g|]}t|tjr|jn|qSrrr%rr.0elemrrr ]sz0_canonicalize_boundary_check..rz'Duplicate dimension in `boundary_check`)hasattrrr rr1setsortedtuple)boundary_check block_shaperrrr_canonicalize_boundary_checkYs *rc Cs||s|rtd|jjj} | tjks,td| rH|tjj krHtd|jj} t || }t | |j|||||| S)NK`mask` and `other` arguments cannot be specified for loading block pointers3`tl.int1` should be rewrited in `tl.make_block_ptr`z@Padding option `nan` is not supported for integer block pointers)r=rG element_tyr%rlr1r<r rrrrr&Zcreate_tensor_pointer_loadrP) ptrmaskrMrrrr is_volatiler"elt_tyrrrr_load_block_pointerfs rc Csn|jjs"td|jd|s2|r2td|s:|rBtd|jsx|rb|jrbtd|rx|jrxtd|jr|rt||j|}|rt||j|}|jj} | j} | t j krt j } t | | j } t|| |}|rt|| |}|jr|j} t | | } n| } |s>t ||j|||| St ||j|j|rZ|jnd|||| SdS)NUnsupported ptr type z in `tl.load`z)`other` cannot be provided without `mask`z`padding_option` or `boundary_check` argument is not supported for loading a tensor ofpointers or loading a scalar. Because the compiler does not know the boundary; please use block pointers (defined by `make_block_ptr`) insteadEMask argument cannot be block type if pointer argument is not a blockzFOther argument cannot be block type if pointer argument is not a block)rGrHrAr=rr}rrrr%rlint8 pointer_type address_spacerIrr&Z create_loadrPZcreate_masked_load) rrrMrrrrrr"ptr_tyrr~rrrr _load_legacy|sH         rzOptional[tl.tensor]r) rrrMrrrrr"r#c Csbt|} t|} t|} |jrF|jjrFt||||| | | || St||||| | | || SdSr$) rrrrGrArr}rr) rrrMrrrrrr"rrrrrrloads rc Cs|r td|jj}|js.t|||}|js@td||jksVtd|jjj|jjksptd|jjj}|tjkstdt ||}t | |j |j |||tj S)Nrz-Value argument must be block type or a scalarz$Block shape and value shape mismatchz2Block element type and value element type mismatchr)r=rGrrr}rr1r%rlrr&Zcreate_tensor_pointer_storerPvoid) rvalrrrrr"rrrrr_store_block_pointers     rc Cs0|jjs"td|jd|r.td|js`|jrJtd|r`|jr`td|jrt||j|}|rt||j|}|jj}|j}|t j krt j }t ||j }t|||}t|||}|st ||j|j||t jS|jjstdt ||j|j|j||t jS)Nrz in `tl.store`z`boundary_check` argument is not supported for storing a tensor of pointers or storing a scalar. Because the compiler does not know the boundary; please use block pointers (defined by `make_block_ptr`) insteadzFValue argument cannot be block type if pointer argument is not a blockrz"Mask must have boolean scalar type)rGrHrAr=rr}rrrr%rlrrrrIr&Z create_storerPrrZcreate_masked_store) rrrrrrr"rrrrr _store_legacys2       r)rrrrrr"r#c CsRt|}t|}|jr:|jjr:t|||||||St|||||||SdSr$)rrrGrArr}rr) rrrrrrr"rrrrrstores r)rcmprrr"r#cCsDt|}|jjj}|jdkr$tdt||j |j |j ||jS)N)rrz9atomic_cas only supports elements with width {16, 32, 64}) rrGrHrrr=r%r&Zcreate_atomic_casrP)rrrrr"rrrr atomic_cas(s   rz&Tuple[tl.tensor, tl.tensor, tl.tensor])rrropr"r#cCs |jjstd|j|jjj}|tjkrJ|dkrJtd|d|tjtj tj tj fkrztd|dt ||j r|rt||j|}|rt||j|}t||jjj|}|s|d}tj}|j r |||j}ttj|j}t||}|||fS)Nz)Pointer argument of store instruction is rTZatomic_z does not support fp16z does not support T)rGrHrAr=rrr%r9rlrZint16r;rr}rrrIZget_int1rrr&)rrrrr"rZmask_irZmask_tyrrratom_red_typechecking_impl4s*      r)rrrrr"r#c CsDt|||d|\}}}t|}|jj}|r|r\t|t j j |j |j |j ||jSt|t j j |j |j |j ||jSt|tj|}t|ttjd|}t|t|d||}t|t|d||} t|t j j |j |j t|||j ||j} t|t j j|j |j t|| |j ||j} t|| | |S)Nmaxr r)rrrGrHr<r]r%r&create_atomic_rmwr ATOMIC_OPMAXrPUMAXrnr'rrget_fp32rrhUMINwhere rrrrr"sca_tyZi_valZi_ptrposnegZpos_retZneg_retrrr atomic_maxPs8 ..rc CsDt|||d|\}}}t|}|jj}|r|r\t|t j j |j |j |j ||jSt|t j j |j |j |j ||jSt|tj|}t|ttjd|}t|t|d||}t|t|d||} t|t j j |j |j t|||j ||j} t|t j j|j |j t|| |j ||j} t|| | |S)Nminr r)rrrGrHr<r]r%r&rr rZMINrPr rnr'rrr rrhr r r rrr atomic_mintsT   rc Cs`t|||d|\}}}t|}|jj}|r6tjjntjj}t | ||j |j |j ||jS)NrT) rrrGrHrBr rZFADDZADDr%r&rrP)rrrrr"rrrrr atomic_adds rc CsDt|||d|\}}}t|}t|tjj|j|j|j||j S)Nand) rrr%r&rr rANDrPrGrrrrr"rrr atomic_andsrc CsDt|||d|\}}}t|}t|tjj|j|j|j||j S)Nor) rrr%r&rr rORrPrGrrrr atomic_orsrc CsDt|||d|\}}}t|}t|tjj|j|j|j||j S)Nxor) rrr%r&rr rZXORrPrGrrrr atomic_xorsrc CsDt|||d|\}}}t|}t|tjj|j|j|j||j S)NZxchg) rrr%r&rr rZXCHGrPrGrrrr atomic_xchgsr)rDrE allow_tf32 out_dtyper"r#c Cs|jr|jst|j|jks>td|jd|jdt|jdks^td|jdt|jdks~td|jd|jdj|jd jkstd|jd |jd |jdjd |jd jd |jd jdkr|jdjdkr|jdjdkstd|jd|jd|jjrj|jjt j ks>td|jdjdksXtd| d }t j }nP|jj s|jjr|d }t j}n"|r|d n|d }|}|jjd }|jjd}||||g}t |||g} t ||j|j||| S)Nz First input (z) and second input (z) must have the same dtype!r zFirst input shape (z) is not two dimensional!zSecond input shape (r rz) and second input shape z= are not compatible for matmul (second index of first shape (z0) must be equal to first index of second shape (rxrz&All values in both first input shape (z) and second input shape (z) must be >= 16!zonly int8 supported!rzsmall blocks not supported!)rGr}r1r0rr~rrHr<r%r get_int32r'r6r:r r7r8Zget_fp16rrr&Z create_dotrP) rDrErr r"r{Z ret_scalar_tyMNr^rrrdots6&  N     r$) conditionxyr"r#cCst|tj|}|jrHt|||\}}t|||\}}t|||\}}t|||dd\}}|jsvt|||\}}|j}t||j |j |j |S)NT) rIr%rlrGr}rFrKr&Z create_selectrP)r%r&r'r"_r^rrrr s  r zSequence[tl.tensor]zTuple[tl.tensor, ...])inputsr!r"r#c sdkrNg}ttD](}|jjg}|t|||qt|ddjj}fddt |DD]}|jj|kstt qtfdd| ddD| tfddttDS) Nrcsg|]\}}|kr|qSrr)rrr)r!rrr!szreduction..cs"rt|}n|}t||Sr$r%rr&r&rWZres_ty)rrr wrap_tensor%szreduction..wrap_tensorcSsg|] }|jqSrrPrtrrrr-sc3s&|]}||jjVqdSr$Z get_resultrGrHrr)r) reduce_opr,rr 1szreduction..) rrrrrrrrGr~rr1Z create_reduceverify) r)r!region_builder_fnr"Z new_inputsrZ new_shaper~r/r)r!r)r2rr,r reductions$   r6csttdkrtddjjfdd|ddD||tfdd ttDS) Nr z7Current implementation only support single tensor inputrcst|}t||Sr$r*r+)r~rrr,Cs z%associative_scan..wrap_tensorcSsg|] }|jqSrr-r.rrrrGsz$associative_scan..c3s&|]}||jjVqdSr$r0r1)r)scan_opr,rrr3Ksz#associative_scan..)rr=rGr~Z create_scanr4rr)r)r!r5r"r)r)r7r~r,rassociative_scan<s    r8z List[str])dtypesr#csfdd}|S)ai We following libdevice's convention to check accepted data types for math functions. It is not a good practice to support all data types as accelerators/GPUs don't support many float16 and bfloat16 math operations. We should let the users know that they are using and invoke explicit cast to convert the data type to the supported one. cstfdd}|S)NcsZt|t|}dd|DD],}|jjjkr"tdd|jjjq"||S)NcSsg|]}t|tjr|qSr)rr%r&)rarrrrbs z@_check_dtype..wrapper..check..zExpected dtype z but got )rvaluesrGrHrr=)argskwargsZall_argsarg)r9fnrrcheck^s z,_check_dtype..wrapper..checkr)r?r@r9)r?rwrapper]sz_check_dtype..wrapperr)r9rBrrAr _check_dtypeUs rC)r&r'r"r#cCs,t|||\}}ddlm}|j|||dSNr )math)Z_builder)rKrEZmulhi)r&r'r"rErrrumulhiks rGZfp32Zfp64rA)r&r"r#cCsddlm}|j||dSrD)rFrErc)r&r"rErrrrcrs rccCst||j|jSr$)r%r&Z create_exprPrGr&r"rrrexpysrIcCst||j|jSr$)r%r&Z create_logrPrGrHrrrlog~srJcCst||j|jSr$)r%r&Z create_cosrPrGrHrrrcossrKcCst||j|jSr$)r%r&Z create_sinrPrGrHrrrsinsrLcCst||j|jSr$)r%r&Z create_sqrtrPrGrHrrrsqrtsrMcCsd|j}|r$t||j|jS|rBt||j|jS| rN|Sds`t d|dS)NFzUnexpected dtype ) r0rBr%r&Z create_fabsrPrGr]Z create_iabsZis_int_unsignedr1)r&r"r0rrrabssrN)r&r;r#cCs:t|jt|krtd|jdt||j|S)NzAShape of input to multiple_of does not match the length of valuesztt.divisibilityrr~r=rPZset_attrr Z make_attrZ get_contextr&r;rrr multiple_ofsrQcCs:t|jt|krtd|jdt||j|S)NzDShape of input to max_contiguous does not match the length of valuesz tt.contiguityrOrPrrrmax_contiguoussrRcCs:t|jt|krtd|jdt||j|S)NzCShape of input to max_constancy does not match the length of valuesz tt.constancyrOrPrrr max_constancysrS)r"r#cCst|tjSr$)r%r&Zcreate_barrierr)r"rrr debug_barriersrTzList[tl.tensor])prefixr<r"r#cCs0g}|D]}||jqt|||tjSr$)rrPr%r&Z create_printr)rUr<r"new_argsr>rrr device_printsrW)condmsg file_namelinenor"r#c CsP|j}|s2t|jd}t||jd|}t||j||||tj S)N)r ) rGr}r%rrHr&rrPZ create_assertr)rXrYrZ func_namer[r"Zcond_tyrrr device_asserts r]cCst|tjr(|r||jS||jSt|tjr|jjdksHtd|j sZtd|j tj kr|r| |j ||j S|j tjkr| |j ||j S|j Sdstdt|dS)Nr z*Expected a scalar in shape/strides/offsetsz8Expected an integer scalar type in shape/strides/offsetsFz3Unsupported element type in shape/strides/offsets: )rr%rrrr!r&rr1r0r<rrrPZ get_int64_tyr]r'Z get_int32_tyrG)r"r require_i64rrr_convert_elem_to_ir_values   r_cs,t|drfdd|DSt|gS)Nrcsg|]}t|qSr)r_rr"r^rrrsz)_convert_to_ir_values..)rr_)r" list_liker^rr`r_convert_to_ir_valuess rb)baser"r#c s:t||}t||}t||dd}|jr8|jjr@td|jjtjkrht|t tj |jj |}t dsxgddDt ddDstdt |ds|g}d d|D}t|ttt|kstd t fd d||||fDstd ||j||||}t|t t|jjS) NFr^zMExpected `base` to be a pointer type (but not a block pointer type or others)rcSs"g|]}t|tjr|jn|qSrrrrrrrsz"make_block_ptr..cSs.g|]&}t|to(d|ko$dknqS)ilrrrrrrszGExpected a list of constant integers (`int32_t` range) in `block_shape`cSs"g|]}t|tjr|jn|qSrrrrrrrszrCrKrTrXrYr\r_rbrdrfrhrirjrorprsrtrurvrwrUrrrrrrrrrrrrrrrrrrFrnrrIrrrrrrrrrrrrrrrrrrrrrr$r r6r8rCrGrcrIrJrKrLrMrNrQrRrSrTrWr]r_rbrjrkrrrrs     &    ;v    9, $.    &'       '