U
    _»{j‘A ã                   @   sø  d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
mZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZ d d	l m!Z!m"Z" d d
l#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z*m+Z+m,Z,m-Z- d dl.m/Z/ d dl0m1Z1m2Z2m3Z3 eeegZ4�ddd„Z5dd„ Z6dd„ Z7dd„ Z8dd„ Z9ej: ;dddg¡dd„ ƒZ<dd „ Z=d!d"„ Z>d#d$„ Z?d%d&„ Z@d'd(„ ZAd)d*„ ZBd+d,„ ZCd-d.„ ZDd/d0„ ZEd1d2„ ZFd3d4„ ZGej: ;d5e Hd d6d d7g¡d d6d7gfe Hd d6d d7g¡dfd8d9d8d:gd8d9d:gfd8d9d8d:gdfg¡d;d<„ ƒZIej: ;d5e Hd d7d d7g¡d d6d7gfe Hd8d=d8d=g¡d8d9d=gfg¡d>d?„ ƒZJej: ;d5e Hd d6d7d7g¡dfd8d9d:d:gdfd d6d7d7gd d6d7gfd8d9d:d:gd8d9d:gfg¡d@dA„ ƒZKej: ;dBdCdDdEg¡dFdG„ ƒZLdHdI„ ZMej: ;dJdKe Hd d6d7d7g¡d d7d gfdKe Hd8d9d:d:g¡d8d8d9gfdLe Hd d7d d7g¡dfdMe Hd8d9d:d:g¡d8d:d9gfdNe Hd d6d7d7g¡d d6gfdNe Hd8d9d:d:g¡d8d9gfdOe Hd d6d7d7g¡d d6d7dPgfdOe Hd8d9d:d:g¡d8d9d:d=gfdQe Hd8d9d:dRg¡d8d9d:gfdQe Hd8d9d:d=g¡d8d9d:gfdQe Hd d6d7dPg¡d d6d7gfg¡ej: ;dSdTdUg¡dVdW„ ƒƒZNej: ;dXdYdZdTd[œfd\dZdUd[œfd]dTg d^œfd_dTd`daœfdbi fg¡dcdd„ ƒZOdedf„ ZPej: ;dge4¡dhdi„ ƒZQej: ;dge4¡djdk„ ƒZRej: ;dge4¡dldm„ ƒZSej: ;dddg¡dndo„ ƒZTdpdq„ ZUej: ;dddg¡drds„ ƒZVdtdu„ ZWdvdw„ ZXdxdy„ ZYdzd{„ ZZd|d}„ Z[d~d„ Z\ej: ;d€d d d6gd d`d6gd gd gfd d d6gd d�d`gd gd gfd d d6gd`d‚d6gd gd gfd d d6gd�d`d‚gd gd gfd d6d gd d`d6gd`gd gfd d6d gd d�d`gd`gd gfd d6d gd`d‚d6gd`gd gfd d6d gd�d`d‚gd`gd gfd d6d6gd d`d6gdƒgd gfd d6d6gd d�d`gd gd gfd d6d6gd`d‚d6gd gd gfd d6d6gd�d`d‚gd gd gfd6d d gd d`d6gd6d6d`gd d6d6gfd6d d gd d�d`gd6d6d`gd d6d6gfd6d d gd`d‚d6gd6d6d`gd d6d6gfd6d d gd�d`d‚gd6d6d`gd d6d6gfd6d d6gd d`d6gd6d6d gd d`d`gfd6d d6gd d�d`gd6d6d gd d`d`gfd6d d6gd`d‚d6gd6d6d gd d`d`gfd6d d6gd�d`d‚gd6d6d gd d`d`gfg¡d„d…„ ƒZ]ej: ;d€d6d gd`d`gd6gd gfd d6gd`d`gd6gd gfd d d6gd�d`d`gd`gd gfd d6d gd�d`d`gd`gd gfd d6d6gd�d`d`gd gd gfd6d d gd�d`d`gd6gd gfd6d d6gd�d`d`gd6gd gfd6d6d gd�d`d`gd6gd gfg¡d†d‡„ ƒZ^dˆd‰„ Z_ej: ;dŠd d�d`d‚d6g¡d‹dŒ„ ƒZ`ej: ;d�d d d d d d6gd d d d d6d6gd d d d6d6d6gd d d6d6d6d6gd d6d6d6d6d6gg¡dŽd�„ ƒZaej: ;d�d d6gd d`d6gd‘fd d6d6gd d`gd‘fd d d gd d`d6gd’fd6d6d6gd d`d6gd’fd“d“d”gd•d–d—gd˜fg¡d™dš„ ƒZbd›dœ„ Zcd�dž„ ZddŸd „ Zed¡d¢„ Zfd£d¤„ Zgd¥d¦„ Zhd§d¨„ Zi�dd«d¬„Zjej: ;d­edehegeef¡ej: ;d®eeif¡d¯d°„ ƒƒZkd±d²„ Zlej: ;d³d´¡ej: ;dµd¶¡ej: ;d·emd6ƒ¡d¸d¹„ ƒƒƒZndºd»„ Zod¼d½„ Zpd¾d¿„ Zqej: ;dÀd6d d6gd�d`d`gfd6d d6gd�d`d`ggfd6d d6ggd�d`d`gfg¡dÁdÂ„ ƒZrdÃdÄ„ ZsdÅdÆ„ ZtdÇdÈ„ ZudÉdÊ„ ZvdËdÌ„ ZwdÍdÎ„ ZxdÏdÐ„ ZydÑdÒ„ ZzdÓdÔ„ Z{ej: ;dÕddg¡dÖd×„ ƒZ|dØdÙ„ Z}dÚdÛ„ Z~dÜdÝ„ ZdÞdß„ Z€ej: ;dàd d6d7dPgd6d�fd d6d7dPgd7d`fd d6d7dPgdPd‚fg¡dádâ„ ƒZ�ej: ;dãe Hdädäd6d6g¡d6d6fe Hdäd6däd6g¡d6d`fe Hdäd6däd6g¡d7d6fe Hd•d•dådåg¡d6d6fe Hd•dåd•dåg¡d6d`fe Hd•dåd•dåg¡d7d6fg¡dædç„ ƒZ‚ej: ;dèe Hd d6d6d7g¡d‚d d6d7dPgfe Hd d6d6d6g¡d`d d6d7dPgfe Hd6d6d6d6g¡d`d d6d7dPgfe Hd8dRdRd8g¡d‚d8d9d=dRgfg¡ej: ;déddg¡dêdë„ ƒƒZƒdìdí„ Z„ej: ;dàd d6d7dPgd6d�fd d6d7dPgd7d`fd d6d7dPgdPd6fg¡dîdï„ ƒZ…ej: ;dðd d6d7dPgdñfd d6d7dPgd©fg¡dòdó„ ƒZ†ej: ;dôd dõd6d7gd•dödågd÷d–d–gd–d÷d–gd÷d`döggddøfd d6d7dPgd•dödågd÷d–d–gd–d÷d–gd÷d`döggddùfd:d:d8d9gd•dödågd÷d–d–gd–d÷d–gd÷d`döggd8d9d:d:gdúfd:d:d8d9gd•dödågd÷d–d–gd–d÷d–gd÷d`döggd8d:d9gdûfd d d6d7gd•dödågd÷d–d–gd–d÷d–gd÷d`döggd d6d7dPgdüfd d d6d7gd•dödågd÷d–d–gd–d÷d–gd÷d`döggd d6dPgdýfd d6gd`d•d•gd–d÷d•ggddþfg¡dÿ�d „ ƒZ‡�d�d„ Zˆej: ;�deeeeg¡ej: ;�d�d�d�d�dg¡�d	�d
„ ƒƒZ‰�d�d„ ZŠdS (  é    N)Ústats)Ú
csr_matrix)ÚdatasetsÚsvm)Úmake_multilabel_classification)ÚUndefinedMetricWarning)ÚLogisticRegression)Úaccuracy_scoreÚaucÚaverage_precision_scoreÚcoverage_errorÚ	dcg_scoreÚ	det_curveÚ%label_ranking_average_precision_scoreÚlabel_ranking_lossÚ
ndcg_scoreÚprecision_recall_curveÚroc_auc_scoreÚ	roc_curveÚtop_k_accuracy_score)Ú_dcg_sample_scoresÚ_ndcg_sample_scores)Útrain_test_split)Úlabel_binarize)Ú_sparse_random_matrix)Úassert_allcloseÚassert_almost_equalÚassert_array_almost_equalÚassert_array_equal)Úsoftmax)Úcheck_arrayÚcheck_consistent_lengthÚcheck_random_stateFc                 C   s  | dkrt  ¡ } | j}| j}|r:||dk  ||dk   }}|j\}}t |¡}tdƒ}| |¡ || ||  }}t	|d ƒ}tj
 d¡}tj|| |d| ¡f }tjdddd�}	|	 |d|… |d|… ¡ ||d… ¡}
|rò|
dd…d	f }
|	 ||d… ¡}||d… }|||
fS )
z¼Make some classification predictions on a toy dataset using a SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    Né   é%   r   éÈ   ZlinearT)ZkernelZprobabilityÚrandom_stateé   )r   Z	load_irisÚdataÚtargetÚshapeÚnpÚaranger"   ÚshuffleÚintÚrandomÚRandomStateZc_Zrandnr   ZSVCÚfitÚpredict_probaZpredict)ZdatasetÚbinaryÚXÚyÚ	n_samplesÚ
n_featuresÚpÚrngZhalfÚclfÚy_scoreÚy_predÚy_true© r>   úa/var/www/html/bwave/python/venv/lib/python3.8/site-packages/sklearn/metrics/tests/test_ranking.pyÚmake_prediction7   s*    


*r@   c                 C   sd   t  | ¡d }|| |k }|| |k }| dd¡| dd¡ }t  |dk¡}|tt|ƒt|ƒ ƒ S )zKAlternative implementation to check for correctness of
    `roc_auc_score`.r'   éÿÿÿÿr   )r+   ÚuniqueÚreshapeÚsumÚfloatÚlen)r=   r;   Ú	pos_labelÚposÚnegZdiff_matrixZ	n_correctr>   r>   r?   Ú_aucg   s    rJ   c           	      C   sª   t  | ¡d }t  | |k¡}t  |¡ddd… }|| }| | } d}tt|ƒƒD ]P}| | |krPd}td|d ƒD ]}| | |krr|d7 }qr||d  }||7 }qP|| S )a>  Alternative implementation to check for correctness of
    `average_precision_score`.

    Note that this implementation fails on some edge cases.
    For example, for constant predictions e.g. [0.5, 0.5, 0.5],
    y_true = [1, 0, 0] returns an average precision of 0.33...
    but y_true = [0, 0, 1] returns 1.0.
    r'   NrA   r   ç      ð?)r+   rB   rD   ZargsortÚrangerF   )	r=   r;   rG   Zn_posÚorderÚscoreÚiÚprecÚjr>   r>   r?   Ú_average_precisionv   s    	

rR   c                 C   sd   t | |ƒ\}}}tt|ƒƒ}tt|ƒƒ}d}tdt|ƒƒD ]$}||| || ||d    7 }q:|S )ao  A second alternative implementation of average precision that closely
    follows the Wikipedia article's definition (see References). This should
    give identical results as `average_precision_score` for all inputs.

    References
    ----------
    .. [1] `Wikipedia entry for the Average precision
       <https://en.wikipedia.org/wiki/Average_precision>`_
    r   r'   )r   ÚlistÚreversedrL   rF   )r=   r;   Ú	precisionÚrecallÚ	thresholdZaverage_precisionrO   r>   r>   r?   Ú_average_precision_slow”   s    
"rX   c                 C   s^   dd„ }|| ||ƒ\}}t ||ƒ}d}|}d||  ||  }	|| }
dd||	 |
|	    S )zcAlternative implementation to check for correctness of `roc_auc_score`
    with `max_fpr` set.
    c                 S   sˆ   t | |ƒ\}}}|||k }t ||¡}|||k }t ||k¡}|d }	||	 || g}
||	 || g}t |t ||
|¡¡}||fS )Nr'   )r   r+   ÚappendÚargmaxÚinterp)r=   Ú	y_predictÚmax_fprÚfprÚtprÚ_Únew_fprÚnew_tprZidx_outZidx_inZx_interpZy_interpr>   r>   r?   Ú_partial_roc¬   s    z,_partial_roc_auc_score.<locals>._partial_rocr   ç      à?r'   )r
   )r=   r\   r]   rc   ra   rb   Zpartial_aucZfpr1Zfpr2Zmin_areaZmax_arear>   r>   r?   Ú_partial_roc_auc_score§   s    
re   ÚdropTc           	      C   sz   t dd�\}}}t||ƒ}t||| d�\}}}t||ƒ}t||dd� t|t||ƒƒ |j|jksft‚|j|jksvt‚d S )NT©r3   ©Zdrop_intermediater#   ©Údecimal)	r@   rJ   r   r
   r   r   r   r*   ÚAssertionError)	rf   r=   r`   r;   Zexpected_aucr^   r_   Ú
thresholdsÚroc_aucr>   r>   r?   Útest_roc_curveÃ   s    

rn   c                  C   sŒ   t j d¡} t  dgd dgd  ¡}| jddd�}t||dd�\}}}|d dksXt‚|d	 dksht‚|j|jksxt‚|j|jksˆt‚d S )
Nr   é2   r'   é   éd   ©ÚsizeTrh   rA   )r+   r/   r0   ÚarrayÚrandintr   rk   r*   )r9   r=   r<   r^   r_   Zthrr>   r>   r?   Útest_roc_curve_end_pointsÑ   s    rv   c            
      C   sŽ   t dd�\} }}t| |ƒ\}}}g }|D ]2}t ||k| @ ¡}t | ¡}	| d| |	 ¡ q(t||dd� |j|jkszt‚|j|jksŠt‚d S )NTrg   rK   r#   ri   )r@   r   r+   rD   rY   r   r*   rk   )
r=   r`   r;   r^   r_   rl   Ztpr_correctÚtÚtpr8   r>   r>   r?   Útest_roc_returns_consistencyÞ   s    
ry   c               	   C   s4   t dd�\} }}t t¡� t| |ƒ W 5 Q R X d S )NFrg   )r@   ÚpytestÚraisesÚ
ValueErrorr   )r=   r`   r;   r>   r>   r?   Útest_roc_curve_multiñ   s    r}   c                  C   s`   t dd�\} }}t| |d ƒ\}}}t||ƒ}t|ddd� |j|jksLt‚|j|jks\t‚d S )NTrg   rd   çÍÌÌÌÌÌì?r#   ri   )r@   r   r
   r   r*   rk   )r=   r`   r;   r^   r_   rl   rm   r>   r>   r?   Útest_roc_curve_confidenceù   s    
r   c                  C   s  t dd�\} }}t | j¡}t| |ƒ\}}}t||ƒ}t|ddd� |j|jksTt‚|j|jksdt‚t | j¡}t| |ƒ\}}}t||ƒ}t|ddd� |j|jks¨t‚|j|jks¸t‚t| |ƒ\}}}t||ƒ}t|ddd� |j|jksðt‚|j|jk�st‚d S )NTrg   rd   r#   ri   gö(\�Âõè?)	r@   r+   Úonesr*   r   r
   r   rk   Úzeros)r=   Úpredr;   Ztrivial_predr^   r_   rl   rm   r>   r>   r?   Útest_roc_curve_hard  s$    


rƒ   c               
   C   s  ddddddddddg
} ddddddddddg
}d}t jt|d�� t| |ƒ\}}}W 5 Q R X t|t t|ƒtj¡ƒ |j	|j	ks†t
‚|j	|j	ks–t
‚d}t jt|d��  tdd„ | D ƒ|ƒ\}}}W 5 Q R X t|t t|ƒtj¡ƒ |j	|j	ksöt
‚|j	|j	k�st
‚d S )Nr'   r   úINo negative samples in y_true, false positive value should be meaningless©ÚmatchúHNo positive samples in y_true, true positive value should be meaninglessc                 S   s   g | ]}d | ‘qS )r'   r>   )Ú.0Úxr>   r>   r?   Ú
<listcomp>4  s     z,test_roc_curve_one_label.<locals>.<listcomp>)rz   Úwarnsr   r   r   r+   ÚfullrF   Únanr*   rk   )r=   r<   Úexpected_messager^   r_   rl   r>   r>   r?   Útest_roc_curve_one_label   s     ÿÿ$r�   c               	   C   sØ  ddg} ddg}t | |ƒ\}}}t| |ƒ}t|dddgƒ t|dddgƒ t|dƒ ddg} ddg}t | |ƒ\}}}t| |ƒ}t|dddgƒ t|dddgƒ t|dƒ ddg} ddg}t | |ƒ\}}}t| |ƒ}t|ddgƒ t|ddgƒ t|dƒ ddg} ddg}t | |ƒ\}}}t| |ƒ}t|dddgƒ t|dddgƒ t|dƒ ddg} ddg}t | |ƒ\}}}t| |ƒ}t|ddgƒ t|ddgƒ t|dƒ ddg} ddg}d}tjt|d	�� t | |ƒ\}}}W 5 Q R X t t¡� t| |ƒ W 5 Q R X t|dddgƒ t|t	j
t	j
t	j
gƒ ddg} ddg}d
}tjt|d	�� t | |ƒ\}}}W 5 Q R X t t¡� t| |ƒ W 5 Q R X t|t	j
t	j
t	j
gƒ t|dddgƒ t	 ddgddgg¡} t	 ddgddgg¡}t t¡� t| |dd� W 5 Q R X t t¡� t| |dd� W 5 Q R X tt| |dd�dƒ tt| |dd�dƒ t	 ddgddgg¡} t	 ddgddgg¡}t t¡� t| |dd� W 5 Q R X t t¡� t| |dd� W 5 Q R X tt| |dd�dƒ tt| |dd�dƒ t	 ddgddgg¡} t	 ddgddgg¡}tt| |dd�dƒ tt| |dd�dƒ tt| |dd�dƒ tt| |dd�dƒ t	 ddgddgg¡} t	 ddgddgg¡}tt| |dd�dƒ tt| |dd�dƒ tt| |dd�dƒ tt| |dd�dƒ d S )Nr   r'   rK   ç        rd   ç      Ð?ç      è?r‡   r…   r„   Úmacro©ÚaverageÚweightedÚsamplesÚmicro)r   r   r   r   rz   r‹   r   r{   r|   r+   r�   rt   )r=   r;   r_   r^   r`   rm   rŽ   r>   r>   r?   Útest_roc_curve_toydata;  s¦    









ÿÿr™   c                  C   s´   ddddddg} ddddddg}t | |d	d
�\}}}t|tjdddgƒ dddddddddddddg} dddddddddddddg}t | |d	d
�\}}}t|tjdddddgƒ d S )Nr   r'   r�   çš™™™™™É?rd   ç333333ã?çffffffæ?rK   Trh   çš™™™™™¹?çš™™™™™é?r~   )r   r   r+   Úinf)r=   r;   r_   r^   rl   r>   r>   r?   Ú test_roc_curve_drop_intermediate£  s    r    c                  C   st   dddddg} dddddg}t  dd	¡}t| ||d
�\}}}t  |¡dk  ¡ dksVt‚t  |¡dk  ¡ dkspt‚d S )Nr   r'   r�   rœ   ç333333Ó?çš™™™™™Ù?rd   rš   é   ©Úsample_weight)r+   Úrepeatr   ÚdiffrD   rk   )r=   r;   r¥   r^   r_   r`   r>   r>   r?   Ú!test_roc_curve_fpr_tpr_increasing±  s    r¨   c                  C   s¬   ddg} ddg}t t| |ƒdƒ ddg} ddg}t t| |ƒdƒ dddg} dddg}t t| |ƒdƒ ddg} ddg}t t| |ƒdƒ dddg} dddg}t t| |ƒdƒ d S ©Nr   r'   rd   )r   r
   )r‰   r5   r>   r>   r?   Útest_auc½  s    



rª   c               	   C   s¤   t  t¡� tdddgddgƒ W 5 Q R X t  t¡� tdgdgƒ W 5 Q R X dddd	g} d
dddg}d t | ¡¡}t jtt |¡d�� t| |ƒ W 5 Q R X d S )Nr�   rd   rK   r�   rš   r#   r'   rp   é   r£   é   é   é   z+x is neither increasing nor decreasing : {}r…   )	rz   r{   r|   r
   Úformatr+   rt   ÚreÚescape)r‰   r5   Úerror_messager>   r>   r?   Útest_auc_errorsÐ  s    r³   zy_true, labelsr'   r#   ÚaÚbÚcc              	   C   s^  t  dddgdddgdddgdd	dgg¡}td
dd
gdddgƒ}tdd
dgdddgƒ}|| d }td
d
dgdddgƒ}tddd
gdddgƒ}|| d }td
dgdd	gƒ}	tdd
gddgƒ}
|	|
 d }|| | d }tt| ||dd�|ƒ |||g}dddg}t j||d�}tt| ||ddd�|ƒ d}tjt|d�� t| ||dd d� W 5 Q R X d S )Nr�   rž   r¡   r¢   çffffffÖ?rd   ç333333Ã?r   rš   r'   r#   rp   Úovo©ÚlabelsÚmulti_classr’   )Úweightsr–   ©r»   r¼   r•   z6average=None is not implemented for multi_class='ovo'.r…   )r+   rt   r   r   r•   rz   r{   ÚNotImplementedError)r=   r»   Úy_scoresÚscore_01Úscore_10Zaverage_score_01Zscore_02Zscore_20Zaverage_score_02Zscore_12Zscore_21Zaverage_score_12Zovo_unweighted_scoreZpair_scoresZ
prevalenceZovo_weighted_scorer²   r>   r>   r?   Ú#test_multiclass_ovo_roc_auc_toydataá  sB    "ÿþ

    ÿürÃ   Údc                 C   sœ   t  dddgdddgdddgdddgg¡}tdd	dd	gddddgƒ}td	dd	dgddddgƒ}|| d
 }tt| ||dd�|ƒ tt| ||ddd�|ƒ d S )Nrš   r�   rž   r›   r¢   gš™™™™™á?gÍÌÌÌÌÌÜ?r'   r   r#   r¹   rº   r–   r¾   ©r+   rt   r   r   )r=   r»   rÀ   rÁ   rÂ   Z	ovo_scorer>   r>   r?   Ú*test_multiclass_ovo_roc_auc_toydata_binary  s(    "ÿ ÿ    ÿürÆ   c                 C   sü   t  dddgdddgdddgdddgg¡}tdd	d	d	g|d d …d	f ƒ}td	dd	d	g|d d …df ƒ}td	d	ddg|d d …d
f ƒ}tt| |d|d d�|||gƒ || | d }tt| |d|d�|ƒ |d |d  |d  }tt| |d|dd�|ƒ d S )NrK   r�   r�   rd   r¢   rž   r¡   r'   r   r#   Úovr)r¼   r»   r•   g      @)r¼   r»   r‘   r–   rÅ   )r=   r»   rÀ   Zout_0Zout_1Zout_2Zresult_unweightedZresult_weightedr>   r>   r?   Ú#test_multiclass_ovr_roc_auc_toydata@  s4    "ÿþþ    ÿürÈ   zmulti_class, average)rÇ   r“   )rÇ   r˜   )r¹   r“   c                 C   sÈ   t  ddddg¡}ddddgddddgddddgdddd	gg}tt||| |d
�dƒ ddddgddddgddddgddddgg}t||| |d
�dk s˜t‚dt  d¡ }t||| |d
�t d¡ksÄt‚d S )Nrp   r'   r#   r   r�   rK   r’   gš™™™™™©?r¸   ©r¼   r•   r‘   )r«   r«   rd   )r+   rt   r   r   rk   r€   rz   Úapprox)r¼   r•   r=   Z	y_perfectZy_imperfectZy_chancer>   r>   r?   Ú0test_perfect_imperfect_chance_multiclass_roc_auck  s8    	



üþ



üÿÿ   ÿþrË   c           	         sŽ   | ‰ t jjdddgdˆ d�}t ‡ fdd„|D ƒ¡}t|dd	d
gd�}t| ¡ | ¡ ƒ\}}}t||ƒ}t	||ddd�}|t
 |¡ksŠt‚d S )Nç       @rK   rd   éè  )rs   r&   c                    s"   g | ]}t jjd |ˆ d� ¡ ‘qS )r'   )Únr8   r&   )r   ZmultinomialÚrvsrZ   )rˆ   Zy_pred_i©Úseedr>   r?   rŠ   �  s   ÿz3test_micro_averaged_ovr_roc_auc.<locals>.<listcomp>r   r'   r#   )ÚclassesrÇ   r˜   rÉ   )r   Z	dirichletrÏ   r+   Úasarrayr   r   Zravelr
   r   rz   rÊ   rk   )	Úglobal_random_seedr<   r=   Zy_onehotr^   r_   r`   Zroc_auc_by_handZroc_auc_autor>   rÐ   r?   Útest_micro_averaged_ovr_roc_auc•  s    
þÿ
rÕ   zmsg, y_true, labelsz!Parameter 'labels' must be uniquezKNumber of classes in y_true not equal to the number of columns in 'y_score'z"Parameter 'labels' must be orderedzMNumber of given labels, 2, not equal to the number of columns in 'y_score', 3zMNumber of given labels, 4, not equal to the number of columns in 'y_score', 3rp   z2'y_true' contains labels not in parameter 'labels'Úer¼   r¹   rÇ   c              	   C   sX   t  dddgdddgdddgdd	dgg¡}tjt| d
�� t||||d� W 5 Q R X d S )Nr�   rž   r¡   r¢   r·   rd   r¸   r   rš   r…   rº   )r+   rt   rz   r{   r|   r   )Úmsgr=   r»   r¼   rÀ   r>   r>   r?   Ú*test_roc_auc_score_multiclass_labels_error©  s
    I"ÿrØ   zmsg, kwargszLaverage must be one of \('macro', 'weighted', None\) for multiclass problemsr—   )r•   r¼   zUaverage must be one of \('micro', 'macro', 'weighted', None\) for multiclass problemszksample_weight is not supported for multiclass one-vs-one ROC AUC, 'sample_weight' must be None in this case)r¼   r¥   z|Partial AUC computation not available in multiclass setting, 'max_fpr' must be set to `None`, received `max_fpr=0.5` insteadrd   )r¼   r]   z'multi_class must be in \('ovo', 'ovr'\)c              	   C   sX   t dƒ}| dd¡}t|ƒ}|jdddd�}tjt| d�� t||f|Ž W 5 Q R X d S )Né”  é   rp   r   rr   r…   )r"   Úrandr   ru   rz   r{   r|   r   )r×   Úkwargsr9   r;   Zy_probr=   r>   r>   r?   Ú#test_roc_auc_score_multiclass_errorú  s    'rÝ   c               
   C   st  t dƒ} |  d¡}tjddd�}d}tjt|d�� t||ƒ W 5 Q R X tjddd�}tjt|d�� t||ƒ W 5 Q R X tj	dddd�}tjt|d�� t||ƒ W 5 Q R X t
jdd	��° t dƒ} |  d¡}tjddd�}tjt|d�� t||ƒ W 5 Q R X tjddd�}tjt|d�� t||ƒ W 5 Q R X tj	dddd�}tjt|d�� t||ƒ W 5 Q R X W 5 Q R X d S )
NrÙ   é
   r.   ©ZdtypezROC AUC score is not definedr…   rA   T)Úrecord)r"   rÛ   r+   r�   rz   r{   r|   r   r€   rŒ   ÚwarningsÚcatch_warnings)r9   r<   r=   Úerr_msgr>   r>   r?   Útest_auc_score_non_binary_class)  s0    

rä   Ú
curve_funcc              	   C   sN   t dƒ}|jdddd�}| d¡}d}tjt|d�� | ||ƒ W 5 Q R X d S )NrÙ   r   rp   rÞ   rr   z"multiclass format is not supportedr…   )r"   ru   rÛ   rz   r{   r|   )rå   r9   r=   r<   r×   r>   r>   r?   Ú&test_binary_clf_curve_multiclass_errorI  s    
ræ   c              	   C   s   d}t jt|d��" | tjddgdd�ddgƒ W 5 Q R X t jt|d��" | tjddgtd�ddgƒ W 5 Q R X d	}t jt|d��" | tjd
dgdd�ddgƒ W 5 Q R X ddddg}| ddddg|ƒ}| ddddg|ƒ}t||ƒD ]\}}tj ||¡ qäd S )Nz‘y_true takes value in {'a', 'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.r…   r´   rµ   z<U1rß   r�   rK   z“y_true takes value in {b'a', b'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.ó   aó   bz<S1rš   gáz®GáÚ?r   r'   )	rz   r{   r|   r+   rt   ÚobjectÚzipZtestingr   )rå   r×   r<   Z	int_curveZfloat_curveZint_curve_partZfloat_curve_partr>   r>   r?   Ú(test_binary_clf_curve_implicit_pos_labelS  s    ÿ&&ÿ&rë   c                 C   s€   dddddg}dddddg}dddddg}| |||d�}| |d d	… |d d	… |d d	… d�}t ||ƒD ]\}}t||ƒ qhd S )
Nr   r'   r�   rš   r¡   r¢   rd   r¤   rA   )rê   r   )rå   r=   r;   r¥   Zresult_1Zresult_2Zarr_1Zarr_2r>   r>   r?   Ú(test_binary_clf_curve_zero_sample_weightw  s    &rì   c           
      C   sj  t dd�\}}}t||| ƒ t|dd … |dd … | d�\}}}|d dksPt‚|d |dd …  ¡ kslt‚d|t |dk¡< | ¡ }t||| ƒ t||ƒ ddddg}ddd	d
g}	t||	| d�\}}}| �rt	|ddddgƒ t	|ddddgƒ t	|ddd
gƒ n:t	|dddddgƒ t	|dddddgƒ t	|ddd	d
gƒ |j
|j
k�sPt‚|j
|j
d k�sft‚d S )NTrg   r'   rh   r   rK   rA   r#   rp   r«   rd   gÚÁQUUÕ?r�   )r@   Ú_test_precision_recall_curver   rk   Úmeanr+   ÚwhereÚcopyr   r   rs   )
rf   r=   r`   r;   r8   Úrrw   Zy_true_copyr»   Zpredict_probasr>   r>   r?   Útest_precision_recall_curve„  s*    $
rò   c                 C   s´   t | ||d�\}}}t| |ƒ}t|ddƒ t|t| |ƒƒ tt| |ƒ|dd� |j|jks^t‚|j|jd ksrt‚t | t 	|¡|d�\}}}|j|jksœt‚|j|jd ks°t‚d S )Nrh   g°rh‘í|ë?rp   r#   ri   r'   )
r   rX   r   r   r   rR   rs   rk   r+   Ú
zeros_like)r=   r;   rf   r8   rñ   rl   Zprecision_recall_aucr>   r>   r?   rí   ¤  s*    
 ÿ  ÿ  ÿrí   c              
   C   s¾  t jdd���4 ddg}ddg}t||| d�\}}}t||ƒ}t|dddgƒ t|dddgƒ t|dƒ ddg}ddg}t||| d�\}}}t||ƒ}t|dddgƒ t|dddgƒ t|dƒ ddg}ddg}t||| d�\}}}t||ƒ}t|ddgƒ t|ddgƒ t|dƒ ddg}ddg}t||| d�\}}}t||ƒ}t|dddgƒ t|dddgƒ t|dƒ ddg}ddg}t||| d�\}}}t||ƒ}t|ddgƒ t|ddgƒ t|dƒ ddg}d	d
g}tjtdd�� t||| d�\}}}W 5 Q R X tjtdd�� t||ƒ}W 5 Q R X t	|dddgƒ t	|dddgƒ t	|dƒ ddg}d	d
g}t||| d�\}}}tt||ƒdƒ t|dddgƒ t|dddgƒ t  
ddgddgg¡}t  
ddgddgg¡}tjtdd�� t	t||dd�dƒ W 5 Q R X tjtdd�� t	t||dd�dƒ W 5 Q R X t	t||dd�dƒ t	t||dd�dƒ t  
ddgddgg¡}t  
ddgddgg¡}tjtdd�� t	t||dd�dƒ W 5 Q R X tjtdd�� t	t||dd�dƒ W 5 Q R X t	t||dd�d
ƒ t	t||dd�dƒ t  
ddgddgg¡}t  
ddgddgg¡}tt||dd�dƒ tt||dd�dƒ tt||dd�dƒ tt||dd�dƒ t  
ddgddgg¡}t  
ddgddgg¡}tjtdd�� t	t||dd�dƒ W 5 Q R X t	t||dd�dƒ tjtdd�� t	t||dd�dƒ W 5 Q R X tjtdd�� t	t||dd�dƒ W 5 Q R X t  
ddgddgg¡}t  
ddgddgg¡}t	t||dd�dƒ t	t||dd�dƒ t	t||dd�dƒ t	t||dd�dƒ t  
ddgddgg¡}t  
ddgddgg¡}tt||dd�dƒ tt||dd�dƒ tt||dd�dƒ tt||dd�dƒ W 5 Q R X t jdd��` t  
ddgddgg¡}t  
ddgddgg¡}tjtdd�� t	t||dd�dƒ W 5 Q R X W 5 Q R X d S )NÚraise)Úallr   r'   rh   rd   rK   r�   r‘   r’   z!No positive class found in y_truer…   r“   r”   r–   r—   r˜   Úignore)r+   Zerrstater   r   r   r   rz   r‹   ÚUserWarningr   rt   )rf   r=   r;   r8   rñ   r`   Zauc_prcr>   r>   r?   Ú#test_precision_recall_curve_toydataº  sD   










 ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿ ÿrø   c                  C   s&  ddddddg} ddddddg}t | |d	d
�\}}}t|dddgƒ dddddddddddddg} dddddddddddddg}t | |d	d
�\}}}t|ddddddgƒ ddddg} ddddg}t | |d	d
�\}}}t|ddgƒ ddddg} ddddg}t | |d	d
�\}}}t|ddddgƒ dS )z9Check the behaviour of the `drop_intermediate` parameter.r   r'   r�   rš   rd   r›   rœ   rK   Trh   r�   rž   r~   r¡   N)r   r   )r=   r;   rU   rV   rl   r>   r>   r?   Ú-test_precision_recall_curve_drop_intermediate^  s@      ÿ  ÿ  ÿ  ÿrù   c                  C   s<   t jdtd�} d| d d d…< t  d¡}t| |ƒdks8t‚d S )Nrq   rß   r'   r«   r‘   )r+   r�   r.   r€   r   rk   ©r=   r;   r>   r>   r?   Ú&test_average_precision_constant_values€  s    
rû   c               	   C   sL   t  ddg¡} t  ddg¡}d}tjt|d�� t| |dd� W 5 Q R X d S )Nr   r'   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r…   r#   ©rG   ©r+   rt   rz   r{   r|   r   ©r=   r<   rã   r>   r>   r?   Ú4test_average_precision_score_binary_pos_label_errorsŽ  s
    rÿ   c               	   C   st   t  ddgddgddgddgg¡} t  ddgddgddgddgg¡}d}tjt|d�� t| |dd	� W 5 Q R X d S )
Nr'   r   r~   r�   rž   rš   znParameter pos_label is fixed to 1 for multilabel-indicator y_true. Do not set pos_label or set pos_label to 1.r…   rü   rý   rþ   r>   r>   r?   Ú8test_average_precision_score_multilabel_pos_label_errors—  s    ""ÿr   c               
   C   s€   t  ddddddg¡} t  dddgdddgddd	gdddgdddgdddgg¡}d
}tjt|d�� t| |dd� W 5 Q R X d S )Nr   r'   r#   rd   rš   r�   r¢   r¡   r›   zdParameter pos_label is fixed to 1 for multiclass y_true. Do not set pos_label or set pos_label to 1.r…   rp   rü   rý   rþ   r>   r>   r?   Ú8test_average_precision_score_multiclass_pos_label_errors¤  s    úÿÿr  c                  C   sÄ   t dd�\} }}t| |ƒ}t| d| ƒ}t| d| ƒ}t| |d ƒ}||ksPt‚||ks\t‚||ksht‚t| |ƒ}t| d| ƒ}t| d| ƒ}	t| |d ƒ}
||ks¨t‚||	ks´t‚||
ksÀt‚d S )NTrg   rq   g�íµ ÷Æ°>rÞ   )r@   r   rk   r   )r=   r`   r;   rm   Zroc_auc_scaled_upZroc_auc_scaled_downZroc_auc_shiftedZpr_aucZpr_auc_scaled_upZpr_auc_scaled_downZpr_auc_shiftedr>   r>   r?   Útest_score_scale_invariance¹  s    

r  z(y_true,y_score,expected_fpr,expected_fnrr‘   r’   r�   c                 C   s(   t | |ƒ\}}}t||ƒ t||ƒ d S ©N©r   r   ©r=   r;   Zexpected_fprZexpected_fnrr^   Úfnrr`   r>   r>   r?   Útest_det_curve_toydataÒ  s    
r  c                 C   s(   t | |ƒ\}}}t||ƒ t||ƒ d S r  r  r  r>   r>   r?   Útest_det_curve_tie_handlingó  s    
r  c                
   C   s>   t tdddgdddgƒtddddddgddddddgƒƒ d S r©   )r   r   r>   r>   r>   r?   Útest_det_curve_sanity_check  s     þr	  r;   c                 C   sN   t ddddddgt d| ¡d�\}}}t|dgƒ t|dgƒ t|| gƒ d S )Nr   r'   r¬   rú   )r   r+   rŒ   r   )r;   r^   r  rW   r>   r>   r?   Útest_det_curve_constant_scores  s     
ÿr
  r=   c                 C   s.   t | | d�\}}}t|dgƒ t|dgƒ d S )Nrú   r   r  )r=   r^   r  r`   r>   r>   r?   Útest_det_curve_perfect_scores  s    r  zy_true, y_pred, err_msgzinconsistent numbers of samplesz Only one class present in y_trueÚcancerú
not cancerrš   r¡   rž   zpos_label is not specifiedc              	   C   s(   t jt|d�� t| |ƒ W 5 Q R X d S )Nr…   )rz   r{   r|   r   rþ   r>   r>   r?   Útest_det_curve_bad_input+  s    r  c            	      C   sº   dgd dgd  } t  ddddd	dd
dddg
¡}d| }t| |dd�\}}}t| |dd�\}}}|d t d¡ksxt‚|d t d¡ksŽt‚t||d d d… ƒ t||d d d… ƒ d S )Nr  rp   r  r­   r�   r¢   r›   rš   r¡   rd   rœ   r~   r'   rü   r   rA   )r+   rt   r   rz   rÊ   rk   r   )	r=   Zy_pred_pos_not_cancerZy_pred_pos_cancerZfpr_pos_cancerZfnr_pos_cancerZth_pos_cancerZfpr_pos_not_cancerZfnr_pos_not_cancerZth_pos_not_cancerr>   r>   r?   Útest_det_curve_pos_label?  s"    ýýr  c                 C   sP  t | ddggddggƒdƒ t | ddggddggƒdƒ t | ddggddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | ddggddggƒdƒ t | ddggddggƒdƒ t | ddggddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒdƒ t | dddggdddggƒd	ƒ t | ddddggddddggƒdƒ d S )
Nr   r'   r‘   r’   rd   gUUUUUUÕ?gªªªªªªê?gªªªªªªâ?çUUUUUUå?)r   ©Ú
lrap_scorer>   r>   r?   Úcheck_lrap_toyY  sj         ÿ ÿ   ÿ         ÿ       ÿ ÿ  r  c                 C   sÊ   t dƒ}tddƒD ]‚}|jd|fd�}t |¡}t d|f¡}| ||ƒdksPt‚| ||ƒdksbt‚t d|f¡}| ||ƒdks‚t‚| ||ƒdkst‚qt| dgdgdgdggdgdgdgdggƒdƒ d S )Nr   r#   r£   r'   rr   rK   rd   )	r"   rL   Úuniformr+   ró   r�   rk   r€   r   )r  r&   Ún_labelsr;   Zy_score_tiesr=   r>   r>   r?   Ú!check_zero_or_all_relevant_labels”  s    
( ÿr  c              	   C   sÜ  t  t¡� | dddgdddgƒ W 5 Q R X t  t¡�. | dddgdddgd	ddgd
ddggƒ W 5 Q R X t  t¡�. | dddgdddgd	ddgd
ddggƒ W 5 Q R X t  t¡�  | ddgddggddgƒ W 5 Q R X t  t¡�" | ddgddggddggƒ W 5 Q R X t  t¡�$ | ddgddggdgdggƒ W 5 Q R X t  t¡�" | ddggddgddggƒ W 5 Q R X t  t¡�$ | dgdggddgddggƒ W 5 Q R X t  t¡�$ | ddgddggdgdggƒ W 5 Q R X d S )Nr   r'   r‘   r¡   rš   r#   r’   r�   rœ   rž   )rz   r{   r|   r  r>   r>   r?   Úcheck_lrap_error_raised«  s*     2 ÿ$&(&(r  c              	   C   sz   t ddƒD ]j}t d|f¡}t d|ƒD ]L}t || ƒD ]:}t d|f¡}d|d||| …f< t| ||ƒ|| ƒ q6q&q
d S )Nr#   rÞ   r'   r   )rL   r+   r€   r�   r   )r  r  r;   Ú
n_relevantrH   r=   r>   r>   r?   Úcheck_lrap_only_tiesÅ  s    r  c              	      sÖ   t ddƒD ]Æ}|t |¡ d|f¡d  }t d|f¡}d|d< d|d< t| ||ƒd| d d ƒ t d|ƒD ]`‰ t |ˆ  ƒD ]N‰t d|f¡}d|dˆˆˆ  …f< t| ||ƒt‡ ‡fdd„t ˆ ƒD ƒƒƒ q~qnq
d S )	Nr#   rÞ   r'   )r   r   )r   rA   r   c                 3   s&   | ]}|d  ˆ| d  ˆ   V  qdS )r'   Nr>   ©rˆ   rñ   ©r  rH   r>   r?   Ú	<genexpr>è  s   ÿz>check_lrap_without_tie_and_increasing_score.<locals>.<genexpr>)rL   r+   r,   rC   r�   r   rD   )r  r  r;   r=   r>   r  r?   Ú+check_lrap_without_tie_and_increasing_scoreÔ  s     þþr  c                    s
  t | |ƒ t| ƒ} t|ƒ}| j\}}t |f¡}t|ƒD ]È}tj|| dd�\}}|j}|| ‰tjˆ|d d� 	¡ }	|	ˆ ‰| |  
¡ d }
|
jdks¢|
j|kr¬d||< q8d||< |
D ]4‰ t‡ ‡fdd„|
D ƒƒ}||  |ˆˆ   7  < q¸||  |
j  < q8| ¡ S )	z8Simple implementation of label ranking average precisionT)Zreturn_inverser'   )Z	minlengthr   r�   c                 3   s   | ]}ˆ| ˆˆ  kV  qd S r  r>   r  ©ÚlabelZrankr>   r?   r    s     z_my_lrap.<locals>.<genexpr>)r!   r    r*   r+   ÚemptyrL   rB   rs   ZbincountZcumsumZnonzerorD   rî   )r=   r;   r6   r  rN   rO   Zunique_rankZinv_rankZn_ranksZ	corr_rankZrelevantZn_ranked_abover>   r  r?   Ú_my_lrapï  s*    

r!  r£   rÚ   c           	      C   sš   t dd|||d�\}}t|jd |jd |d�}t|dƒrB| ¡ }t||ƒ}t||ƒ}t||ƒ t|ƒ}|j	||fd�}t||ƒ}t||ƒ}t||ƒ d S )Nr'   F)r7   Zallow_unlabeledr&   Ú	n_classesr6   r   )Zn_componentsr7   r&   Útoarrayrr   )
r   r   r*   Úhasattrr#  r   r!  r   r"   r  )	r  r"  r6   r&   r`   r=   r;   Z
score_lrapZscore_my_lrapr>   r>   r?   Ú%check_alternative_lrap_implementation  s,    û
	ý





r%  ÚcheckÚfuncc                 C   s   | |ƒ d S r  r>   )r&  r'  r>   r>   r?   Útest_label_ranking_avp5  s    r(  c                   C   s   t tƒ d S r  )r  r   r>   r>   r>   r?   Útest_lrap_error_raisedC  s    r)  r6   )r'   r#   r®   rÚ   r"  )r#   r£   rÞ   r&   c                 C   s   t t|| |ƒ d S r  )r%  r   )r6   r"  r&   r>   r>   r?   Ú$test_alternative_lrap_implementationG  s       ÿr*  c                  C   s    t jddddgddddgddddggtd�} t  ddddgddddgddddgg¡}t  dd	d
g¡}t  d
d
dg¡}tt| ||d�t  || ¡t  |¡ ƒ d S )Nr'   r   rß   r¡   r¢   rš   r�   rd   r’   rK   r�   r¤   )r+   rt   Úboolr   r   rD   )r=   r;   Zsamplewise_lrapsr¥   r>   r>   r?   Ú&test_lrap_sample_weighting_zero_labelsP  s    , ÿ  ÿür,  c                   C   s$  t tddggddggƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddgdddggdd	d
gdddggƒdƒ t tdddgdddgdddggddd
gdddgdddggƒdƒ t tdddgdddgdddggddd
gdddgdddggƒdƒ d S )Nr   r'   r‘   r’   r#   rd   rp   r�   ç      $@éýÿÿÿrÌ   rÞ   g«ªªªªª@©r   r   r>   r>   r>   r?   Útest_coverage_errore  s\                            (þ ÿü ÿür0  c                   C   st  t tddggddggƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ d S )Nr   rd   r'   r#   r‘   rp   r/  r>   r>   r>   r?   Útest_coverage_tie_handlingœ  s           r1  zy_true, y_scorec              	   C   s(   t jtdd�� t| |ƒ W 5 Q R X d S )Nz'Expected 2D array, got 1D array insteadr…   )rz   r{   r|   r   rú   r>   r>   r?   Útest_coverage_1d_error_message¬  s    r2  c                	   C   sÖ  t tddggddggƒdƒ t tddggddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddgdddggddd	gddd
ggƒdƒ t tdddgdddgdddggddd	gddd
gdddggƒdƒ t tdddgdddgdddggddd	gd
dd
gdddggƒdƒ t ttt dddgdddgg¡ƒddd	gd
dd
ggƒdƒ d S )Nr   r'   r‘   r’   rd   rK   r�   r-  r.  rp   rÞ   r#   )r   r   r   r+   rt   r>   r>   r>   r?   Útest_label_ranking_loss»  sR              (þ ÿü ÿü ÿür3  c                	   C   s4  t  t¡�  tddgddggddgƒ W 5 Q R X t  t¡�" tddgddggddggƒ W 5 Q R X t  t¡�$ tddgddggdgdggƒ W 5 Q R X t  t¡�" tddggddgddggƒ W 5 Q R X t  t¡�$ tdgdggddgddggƒ W 5 Q R X t  t¡�$ tddgddggdgdggƒ W 5 Q R X d S )Nr   r'   )rz   r{   r|   r   r>   r>   r>   r?   Ú$test_ranking_appropriate_input_shapeî  s    $&(&(r4  c                   C   sü   t tddggddggƒdƒ t tddggddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ t tdddggdddggƒdƒ d S )Nr'   r   rd   r‘   )r   r   r>   r>   r>   r?   Útest_ranking_loss_ties_handlingþ  s         r5  c                  C   sH   t ddd�\} }| d }t||ƒ tj d¡ d¡\}}t||ƒ d S ©Nr   rÞ   )r&   r"  r'   )r#   rq   rÞ   )r   Ú_test_dcg_score_forr+   r/   r0   Úrandom_sample©r`   r=   r;   r>   r>   r?   Útest_dcg_score
  s
    

r:  c                 C   s¾   t  t  | jd ¡d ¡}t| | ƒ}t| |ƒ}||k ¡ s>t‚t| | dd�|k ¡ sXt‚|j| jd fksnt‚|j| jd fks„t‚|t t  	| ¡d d …d d d…f | j
dd�¡ksºt‚d S )Nr'   r#   r£   ©Úkr   rA   ©Zaxis)r+   Úlog2r,   r*   r   rõ   rk   rz   rÊ   ÚsortrD   )r=   r;   ZdiscountÚidealrN   r>   r>   r?   r7    s    

r7  c               	   C   sN  t  t  d¡g¡} t  | j¡}t| |ƒ}t| |dd�}dt  t  dd¡¡ }|t | 	¡ |  
¡  g¡kslt‚|t || d d …d d d…f   	¡ g¡ksšt‚d|dd	d …f< t| |ƒ}t| |dd�}|t || d d …d d d…f   	¡ g¡ksðt‚|t |d d…  	¡ | dd	d …f  
¡  |dd …  	¡ | dd d	…f  
¡   g¡k�sJt‚d S )
Nr£   T©Úignore_tiesr'   r#   r­   rA   r   rp   )r+   rÓ   r,   r�   r*   r   r>  rz   rÊ   rD   rî   rk   )r=   r;   ZdcgZdcg_ignore_tiesZ	discountsr>   r>   r?   Útest_dcg_ties  s"    
 .
.""ÿÿÿrC  c               	   C   s>   t  d¡ d¡} t| | ddd�t t| | ddd�¡ks:t‚d S )Né   )r#   r¬   rp   T)r<  rB  )r+   r,   rC   r   rz   rÊ   rk   )r´   r>   r>   r?   Útest_ndcg_ignore_ties_with_k1  s    ÿrE  c               	   C   sf   t  dddddgg¡} t  dddd	d
gg¡}d}tjt|d�� t| |ƒt d¡ksXt‚W 5 Q R X d S )Ng{®Gázì¿gö(\�Âõà¿g®GázÞ¿gö(\�ÂõØ?gìQ¸…ëá?gìQ¸…ë±?g×£p=
×Ó?r’   g…ëQ¸Õ?gHáz®GÑ?z�ndcg_score should not be used on negative y_true values. ndcg_score will raise a ValueError on negative y_true values starting from version 1.4.r…   g�€&Â†Àx@)r+   rt   rz   r‹   ÚFutureWarningr   rÊ   rk   )r=   r;   rŽ   r>   r>   r?   Útest_ndcg_negative_ndarray_warn9  s    ÿrG  c                  C   s�   t  d¡ dd¡} | t j d¡jdd| jd� }t| |ƒ}t| |dd	�}|t 	|¡ksZt
‚|t 	d
¡kslt
‚|d7 }t| |ƒt 	d
¡ksŒt
‚d S )NéF   r­   rÞ   r   çš™™™™™É¿rš   rr   TrA  rK   rÍ   )r+   r,   rC   r/   r0   r  r*   r   rz   rÊ   rk   )r=   r;   ZndcgZndcg_no_tiesr>   r>   r?   Útest_ndcg_invariantD  s    
rJ  rB  c              
   C   s"  dt  d¡d d…  }t  t  ddd¡d¡}|t j d¡jdd	|jd
� }t||| d�t	 
dt  t  dd¡¡ ¡ksxt‚t||| d�t	 
dt  t  dd¡¡ ¡ks¦t‚t||| d�t	 
dt  t  dd¡¡ ¡ksÔt‚t||d| d�t	 
dt  t  dd¡¡ ¡k�st‚t||| d�t	 
dt  t  dd¡¡  ¡ ¡k�s:t‚t||| d�t	 
dt  t  dd¡¡  ¡ ¡k�snt‚dt  d¡ }dt  t  dd¡¡  ¡ }t||| d�t	 
|t  d¡ ¡k�s¾t‚t||| d�t	 
t  d¡¡k�sât‚t||| d�t	 
|¡k�s t‚t||| d�t	 
d¡k�st‚d S )Nrp   r­   r£   r¬   rA   )r£   r'   r   rI  rš   rr   rA  r#   r'   rÞ   )Zlog_baserB  )r£   r­   é	   rK   )r+   ÚeyeZtiler,   r/   r0   r  r*   r   rz   rÊ   r>  rk   r   Úlog10r   rî   r   r€   rD   )rB  r=   r;   Zy_score_noisyZexpected_dcg_scorer>   r>   r?   Útest_ndcg_toy_examplesO  s|      ÿ  ÿþ  ÿþ  ÿþ   ÿþ
ÿÿ  ÿþ
  ÿþ
ÿrN  c               	   C   s4   d} t jt| d�� tdggdggƒ W 5 Q R X dS )zhCheck that we raise an informative error message when trying to
    compute NDCG with a single document.zTComputing NDCG is only meaningful when there is more than 1 document. Got 1 instead.r…   r'   N)rz   r{   r|   r   )rã   r>   r>   r?   Útest_ndcg_error_single_documentv  s    ÿrO  c                  C   sH   t ddd�\} }| d }t||ƒ tj d¡ d¡\}}t||ƒ d S r6  )r   Ú_test_ndcg_score_forr+   r/   r0   r8  r9  r>   r>   r?   Útest_ndcg_score�  s
    

rQ  c                 C   sü   t | | ƒ}t | |ƒ}||k ¡ s$t‚| dkjdd�}||  t t |  ¡ ¡¡ksXt‚|| t t | ¡ ¡¡ksxt‚||  t t	| |ƒ|  t	| | ƒ|   ¡ks¬t‚|| t t | ¡ ¡¡ksÌt‚|j
| j
d fksât‚|j
| j
d fksøt‚d S )Nr   r'   r=  )r   rõ   rk   rz   rÊ   r+   r€   rD   r�   r   r*   )r=   r;   r@  rN   Zall_zeror>   r>   r?   rP  ‰  s    

$ ÿÿ
 rP  c               	   C   sJ  t  ddddg¡} t| | dd�dks(t‚t| | dd�dks>t‚t t¡� t| | dd�s\t‚W 5 Q R X t t¡� t| | dd�s„t‚W 5 Q R X t t¡� t| | dd�s¬t‚W 5 Q R X t  ddddg¡}t| |dd�}t| |ƒ}||ksìt‚t| |d	d�d
k�st‚tdd�\} }}t  ddd¡D ]"}t	t| ||d�t
| ||ƒƒ �q"d S )Nr   r'   )r]   gü©ñÒMbP?gš™™™™™¹¿gš™™™™™ñ?r�   g{®Gáz„?r¡   rd   Trg   g-Cëâ6?r£   )r+   rt   r   rk   rz   r{   r|   r@   Zlinspacer   re   )r=   rÀ   Zroc_auc_with_max_fpr_oneZunconstrained_roc_aucr<   r`   r]   r>   r>   r?   Útest_partial_roc_auc_score™  s(    

þrR  zy_true, k, true_scorec              	   C   sV   t  ddddgddddgddddgddddgg¡}t| ||d�}|t |¡ksRt‚d S )Nr¢   r¡   rš   r�   r;  ©r+   rt   r   rz   rÊ   rk   )r=   r<  Ú
true_scorer;   rN   r>   r>   r?   Útest_top_k_accuracy_score³  s    	



üÿrU  zy_score, k, true_scorerA   rœ   c                 C   s„   ddddg}|   ¡ dkr(|  ¡ dkr(dnd}|dkrD| |k tj¡n|}t|| |d�}t||ƒ}||  krzt |¡ks€n t	‚d S )Nr   r'   rd   r;  )
ÚminÚmaxZastyper+   Zint64r   r	   rz   rÊ   rk   )r;   r<  rT  r=   rW   r<   rN   Z	score_accr>   r>   r?   Ú test_top_k_accuracy_score_binaryÈ  s     
rX  zy_true, true_score, labelsÚlabels_as_ndarrayc              	   C   sf   |rt  |¡}t  ddddgddddgddddgddddgg¡}t| |d|d�}|t |¡ksbt‚dS )z,Test when labels and y_score are multiclass.r¢   r¡   rš   r�   r#   ©r<  r»   N)r+   rÓ   rt   r   rz   rÊ   rk   )r=   rT  r»   rY  r;   rN   r>   r>   r?   Ú0test_top_k_accuracy_score_multiclass_with_labelsß  s    




üÿ	r[  c                     s–   t jddddd�\‰ ‰tˆ ˆdd�\} }}}tdd�‰ˆ | |¡ t| |f||fƒD ]<\‰ ‰‡ ‡‡fdd„tddƒD ƒ}t t 	|¡dk¡sTt
‚qTd S )	NrÞ   rÍ   r   )r"  r6   Zn_informativer&   )r&   c                    s    g | ]}t ˆˆ ˆ ¡|d �‘qS )r;  )r   r2   )rˆ   r<  ©r4   r:   r5   r>   r?   rŠ     s    z8test_top_k_accuracy_score_increasing.<locals>.<listcomp>r#   )r   Zmake_classificationr   r   r1   rê   rL   r+   rõ   r§   rk   )ZX_trainZX_testZy_trainZy_testZscoresr>   r\  r?   Ú$test_top_k_accuracy_score_increasingü  s       ÿ

ÿr]  c              	   C   sR   t  ddddgddddgddddgddddgg¡}t| ||d�t |¡ksNt‚d S )Nr£   r­   r   r'   rp   r;  rS  )r=   r<  rT  r;   r>   r>   r?   Útest_top_k_accuracy_score_ties  s    




üÿr^  z	y_true, kr«   c              	   C   sn   t  ddddgddddgddddgddddgg¡}d}tjt|d�� t| ||d�}W 5 Q R X |dksjt‚d S )	Nr¢   r¡   rš   r�   zu'k' \(\d+\) greater than or equal to 'n_classes' \(\d+\) will result in a perfect score and is therefore meaningless.r…   r;  r'   )r+   rt   rz   r‹   r   r   rk   )r=   r<  r;   rŽ   rN   r>   r>   r?   Ú!test_top_k_accuracy_score_warning$  s    



üÿ	ÿr_  zy_true, y_score, labels, msgg=
×£p=â?r�   r¢   z9y type must be 'binary' or 'multiclass', got 'continuous'zZNumber of classes in 'y_true' \(4\) not equal to the number of classes in 'y_score' \(3\).z"Parameter 'labels' must be unique.z#Parameter 'labels' must be ordered.zSNumber of given labels \(4\) not equal to the number of classes in 'y_score' \(3\).z3'y_true' contains labels not in parameter 'labels'.z}`y_true` is binary while y_score is 2d with 3 classes. If `y_true` does not contain all the labels, `labels` must be providedc              	   C   s.   t jt|d�� t| |d|d� W 5 Q R X d S )Nr…   r#   rZ  )rz   r{   r|   r   )r=   r;   r»   r×   r>   r>   r?   Útest_top_k_accuracy_score_error=  s    Sr`  c                  C   sR   t dddgdddggƒ} t dddgdddgg¡}t| |ƒ}|t d¡ksNt‚d S )Nr'   r   rd   r~   r›   r  )r   r+   rt   r   rz   rÊ   rk   )r=   r;   Úresultr>   r>   r?   ÚOtest_label_ranking_avg_precision_score_should_allow_csr_matrix_for_y_true_input”  s    
rb  ÚmetricrÒ   )FT)r   r'   )r�   rK   )ÚzeroÚonec                 C   s    t j d¡}d|d  }}|j||dd�}| |¡}| |||d�}t|tƒr\t  |¡rœt‚n@|\}}	}
t  |¡ 	¡ rxt‚t  |	¡ 	¡ rŠt‚t  |
¡ 	¡ rœt‚dS )zÀCheck that the metric works with different types of `pos_label`.

    We can expect `pos_label` to be a bool, an integer, a float, a string.
    No error should be raised for those types.
    é*   rÞ   rA   T)rs   Úreplacerü   N)
r+   r/   r0   ÚchoicerÛ   Ú
isinstancerE   Úisnanrk   Úany)rc  rÒ   r9   r6   rG   r=   Zy_probara  Zmetric_1Zmetric_2rl   r>   r>   r?   Ú#test_ranking_metric_pos_label_types�  s    


rl  c                 C   sL   t j | ¡}|jdddd�}| d¡}t||ƒ\}}}t  |d ¡sHt‚dS )z¶Check that thresholds do not exceed 1.0 when `y_score` is a probability
    estimate.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/26193
    r   r#   rÞ   rr   N)r+   r/   r0   ru   rÛ   r   Úisinfrk   )rÔ   r9   r=   r;   r`   rl   r>   r>   r?   Ú(test_roc_curve_with_probablity_estimates·  s
    
rn  )NF)r£   rÚ   r   )‹r°   rá   Únumpyr+   rz   Zscipyr   Zscipy.sparser   Zsklearnr   r   Zsklearn.datasetsr   Zsklearn.exceptionsr   Zsklearn.linear_modelr   Zsklearn.metricsr	   r
   r   r   r   r   r   r   r   r   r   r   r   Zsklearn.metrics._rankingr   r   Zsklearn.model_selectionr   Zsklearn.preprocessingr   Zsklearn.random_projectionr   Zsklearn.utils._testingr   r   r   r   Zsklearn.utils.extmathr   Zsklearn.utils.validationr    r!   r"   ZCURVE_FUNCSr@   rJ   rR   rX   re   ÚmarkZparametrizern   rv   ry   r}   r   rƒ   r�   r™   r    r¨   rª   r³   rt   rÃ   rÆ   rÈ   rË   rÕ   rØ   rÝ   rä   ræ   rë   rì   rò   rí   rø   rù   rû   rÿ   r   r  r  r  r  r	  r
  r  r  r  r  r  r  r  r  r!  r%  r(  r)  rL   r*  r,  r0  r1  r2  r3  r4  r5  r:  r7  rC  rE  rG  rJ  rN  rO  rQ  rP  rR  rU  rX  r[  r]  r^  r_  r`  rb  rl  rn  r>   r>   r>   r?   Ú<module>   s  <
ý0
hüþ	
2þþ
üþ	
"ýþ
"ýú	ýú
ú

ú

ú	ýýýÂþG	û	û	û	úâþ#
 
	
#


 $"	""""""""ìþ
øþ


ûþ

ýûþ
;'    ÿ
 üþ	7ýþ
3
&ýþ
úþ
üþ	ýþ
þþ

ü÷
ü÷
ü
÷
ü÷
ü
÷
ü÷û»þR	 
ÿ ÿ