
    Mpjx                       S SK r S SKrS SKrS SKrS SKJr  S SKJr  S SK	J
r
  S SKJr  S SKJr  S SKJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJ r J!r!  S SK"J#r#J$r$J%r%  S S	K&J'r'  S S
K(J)r)  S SK*J+r+  S SK,J-r-J.r.J/r/J0r0J1r1  S SK2J3r3  S SK4J5r5  S SK6J7r7J8r8J9r9  \\\\ /r:GSS jr;S r<S r=S r>S r?\R                  R                  SSS/5      S 5       rBS rCS rDS rES rFS rGS rHS rIS  rJS! rKS" rLS# rM\R                  R                  S$\R                  " / S%Q5      / S&Q4\R                  " / S%Q5      S4/ S'Q/ S(Q4/ S'QS4/5      S) 5       rO\R                  R                  S$\R                  " / S*Q5      / S&Q4\R                  " / S+Q5      / S,Q4/5      S- 5       rP\R                  R                  S$\R                  " / S.Q5      S4/ S/QS4/ S.Q/ S&Q4/ S/Q/ S(Q4/5      S0 5       rQ\R                  R                  S1/ S2Q5      S3 5       rRS4 rS\R                  R                  S5S6\R                  " / S.Q5      / S7Q4S6\R                  " / S/Q5      / S8Q4S9\R                  " / S*Q5      S4S:\R                  " / S/Q5      / S;Q4S<\R                  " / S.Q5      S S=/4S<\R                  " / S/Q5      S>S?/4S@\R                  " / S.Q5      / SAQ4S@\R                  " / S/Q5      / SBQ4SC\R                  " / SDQ5      / S(Q4SC\R                  " / SBQ5      / S(Q4SC\R                  " / SAQ5      / S&Q4/5      \R                  R                  SESFSG/5      SH 5       5       rT\R                  R                  SISJSKSFSL.4SMSKSGSL.4SNSF/ SO.4SPSFSQSR.4SS0 4/5      ST 5       rUSU rVSV rWSW rXSX rYSY rZ\R                  R                  SZ\:5      S[ 5       r[\R                  R                  SZ\:5      S\ 5       r\\R                  R                  S]5      \R                  R                  SZ\\ /5      \R                  R                  S^S_S`/5      Sa 5       5       5       r^\R                  R                  SZ\:5      Sb 5       r_\R                  R                  SSS/5      Sc 5       r`Sd ra\R                  R                  SSS/5      Se 5       rbSf rcSg rdSh reSi rfSj rgSk rhSl ri\R                  R                  Sm/ SnQ/ SoQS /S /4/ SnQ/ SpQS /S /4/ SnQ/ SqQS /S /4/ SnQ/ SrQS /S /4/ SsQ/ SoQ/ StQ/ SuQ4/ SsQ/ SpQ/ StQ/ SuQ4/ SsQ/ SqQ/ StQ/ SuQ4/ SsQ/ SrQ/ StQ/ SuQ4/ SuQ/ SoQSv/S /4/ SuQ/ SpQS /S /4/ SuQ/ SqQS /S /4/ SuQ/ SrQS /S /4/ SwQ/ SoQ/ SxQ/ SyQ4/ SwQ/ SpQ/ SxQ/ SyQ4/ SwQ/ SqQ/ SxQ/ SyQ4/ SwQ/ SrQ/ SxQ/ SyQ4/ SzQ/ SoQ/ S{Q/ S|Q4/ SzQ/ SpQ/ S{Q/ S|Q4/ SzQ/ SqQ/ S{Q/ S|Q4/ SzQ/ SrQ/ S{Q/ S|Q4/5      S} 5       rj\R                  R                  / S~Q/ SwQ/ SoQ/ SQ/ SyQS4/ SwQ/ SoQ/ SQ/ SuQS4/ SwQ/ SpQ/ SQ/ SyQS4/ SwQ/ SpQ/ SQ/ SuQS4/ SzQ/ SoQ/ S{Q/ S|QS4/ SzQ/ SoQ/ S{Q/ S|QS4/ SzQ/ SpQ/ S{Q/ S|QS4/ SzQ/ SpQ/ S{Q/ S|QS4/5      S 5       rk\R                  R                  SmS=S /SQSQ/S=S /S S=/4S S=/SQSQ/S=S /S S=/4/ SnQ/ SQSQS /S S=/4/ SsQ/ SQSQS /S S=/4/ SuQ/ SQS /S /4/ SwQ/ SQ/ S{Q/ SuQ4/ SzQ/ SQ/ S{Q/ SoQ4/ S{Q/ SQ/ S{Q/ SoQ4/5      S 5       rlS rm\R                  R                  S/ SQ5      S 5       rn\R                  R                  S/ SQ/ SQ/ SQ/ SQ/ SQ/5      S 5       ro\R                  R                  SS S=// SoQS4/ SuQS SQ/S4/ SQ/ SoQS4/ SQ/ SoQS4/ SQ/ SQS4/5      S 5       rpS rqS rrS rsS rtS ruS rvS rw GSS jrx\R                  R                  S\r\v\u\s45      \R                  R                  S\\w45      S 5       5       ryS rz\R                  R                  SS5      \R                  R                  SS5      \R                  R                  S\{" S=5      5      S 5       5       5       r|S r}S r~S r\R                  R                  S/ SzQ/ SQ4/ SzQ/ SQ/4/ SzQ// SQ4/5      S 5       rS r\R                  R                  S\55      S 5       rS rS rS rS rS rS rS rS r\R                  R                  SSS/5      S 5       rS rS rS rS r\R                  R                  S/ SAQS=S4/ SAQSSQ4/ SAQSS4/5      S 5       r\R                  R                  S\R                  " / SQ5      S=S=4\R                  " / SQ5      S=SQ4\R                  " / SQ5      SS=4\R                  " / SQ5      S=S=4\R                  " / SQ5      S=SQ4\R                  " / SQ5      SS=4/5      S 5       r\R                  R                  S\R                  " / SQ5      S/ SAQ4\R                  " / SyQ5      SQ/ SAQ4\R                  " / SQ5      SQ/ SAQ4\R                  " / SQ5      S/ SQ4/5      \R                  R                  SSS/5      S 5       5       rS r\R                  R                  S/ SAQS=S4/ SAQSSQ4/ SAQSS=4/5      S 5       r\R                  R                  S/ SAQS4/ SAQS4/5      S 5       r\R                  R                  S/ SQ/ SQ/ SQ/ SQ/ SQ/SS4/ SAQ/ SQ/ SQ/ SQ/ SQ/SS4/ SQ/ SQ/ SQ/ SQ/ SQ// S/QS4/ SQ/ SQ/ SQ/ SQ/ SQ// S;QS4/ SQ/ SQ/ SQ/ SQ/ SQ// SAQS4/ SQ/ SQ/ SQ/ SQ/ SQ// SQS4S S=// SQ/ SQ/SS4/5      S 5       r\R                  R                  S\55      S 5       r\R                  R                  S\\\\ /5      \R                  R                  S/ SQ5      S 5       5       rS rGSS jr\R                  R                  S\\\\/5      \R                  R                  SS\R                  " / SQ5      /5      S 5       5       rS rS r\R                  R                  SSS/5      S 5       r\R                  R                  SS S=/5      S 5       rS rS rS rGS  rGS rg(      N)stats)datasets)make_multilabel_classification)UndefinedMetricWarning)LogisticRegression)accuracy_scoreaucaverage_precision_scoreconfusion_matrixconfusion_matrix_at_thresholdscoverage_error	dcg_score	det_curve%label_ranking_average_precision_scorelabel_ranking_lossmetric_at_thresholds
ndcg_scoreprecision_recall_curveprecision_scorerecall_scoreroc_auc_score	roc_curvetop_k_accuracy_score)_dcg_sample_scores_ndcg_sample_scores2_sort_inputs_and_compute_classification_thresholds)train_test_split)label_binarize)_sparse_random_matrix)_convert_containerassert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equal)softmax)CSR_CONTAINERS)check_arraycheck_consistent_lengthcheck_random_stateFc                 z   U c  [         R                  " 5       n U R                  nU R                  nU(       a  X#S:     X3S:     p2UR                  u  pE[
        R                  " U5      n[        S5      nUR                  U5        X&   X6   p2[        US-  5      n[
        R                  R                  S5      n[
        R                  X'R                  USU-  5      4   n[        SS9n	U	R                  USU USU 5      R!                  X(S 5      n
U(       a	  U
SS2S4   n
U	R#                  X(S 5      nX8S nXU
4$ )zMake some classification predictions on a toy dataset using an SVC

If binary is True restrict to a binary classification problem instead of a
multiclass classification problem
N   %   r      random_state   )r   	load_irisdatatargetshapenparanger)   shuffleintrandomRandomStatec_randnr   fitpredict_probapredict)datasetbinaryXy	n_samples
n_featuresprnghalfclfy_scorey_predy_trues                ^/var/www/html/pdf-tiff/venv/lib/python3.13/site-packages/sklearn/metrics/tests/test_ranking.pymake_predictionrN   B   s-    $$&AAQxq51GGI
		)A
R
 CKKN4qy1}D ))


"C
a9cJ&6778A !
,Cggah%4)77%AG !Q$-[[5"FuXF7""    c                    [         R                  " U 5      S   nXU:H     nXU:g     nUR                  SS5      UR                  SS5      -
  n[         R                  " US:  5      nU[	        [        U5      [        U5      -  5      -  $ )zGAlternative implementation to check for correctness of
`roc_auc_score`.r0   r   )r5   uniquereshapesumfloatlen)rL   rJ   	pos_labelposnegdiff_matrix	n_corrects          rM   _aucr\   r   s     		&!!$I I%
&C
I%
&C++a$s{{2q'99K{Q'IuSXC0111rO   c                 f   [         R                  " U 5      S   n[         R                  " X:H  5      n[         R                  " U5      SSS2   nX   nX   n Sn[	        [        U5      5       H?  nX   U:X  d  M  Sn[	        SUS-   5       H  nX   U:X  d  M  US-  nM     XvS-   -  nXW-  nMA     XS-  $ )a&  Alternative implementation to check for correctness of
`average_precision_score`.

Note that this implementation fails on some edge cases.
For example, for constant predictions e.g. [0.5, 0.5, 0.5],
y_true = [1, 0, 0] returns an average precision of 0.33...
but y_true = [0, 0, 1] returns 1.0.
r0   NrQ   r         ?)r5   rR   rT   argsortrangerV   )	rL   rJ   rW   n_posorderscoreiprecjs	            rM   _average_precisionrg      s     		&!!$IFF6&'EJJw"%EnG]FE3w< 9	! D1a!e_9	)CKD % GODME ! =rO   c                     [        X5      u  p#n[        [        U5      5      n[        [        U5      5      nSn[        S[	        U5      5       H  nXRU   X6   X6S-
     -
  -  -  nM     U$ )aS  A second alternative implementation of average precision that closely
follows the Wikipedia article's definition (see References). This should
give identical results as `average_precision_score` for all inputs.

References
----------
.. [1] `Wikipedia entry for the Average precision
   <https://en.wikipedia.org/wiki/Average_precision>`_
r   r0   )r   listreversedr`   rV   )rL   rJ   	precisionrecall	thresholdaverage_precisionrd   s          rM   _average_precision_slowro      sq     $:&#J IyXi()I(6"#F1c)n%q\VYA-FGG &rO   c                 z    S nU" XU5      u  pE[        XE5      nSnUnSX-
  -  X-   -  n	X-
  n
SSXi-
  X-
  -  -   -  $ )z[Alternative implementation to check for correctness of `roc_auc_score`
with `max_fpr` set.
c                    [        X5      u  p4nX3U:*     n[        R                  " Xb5      nXCU:*     n[        R                  " X2:  5      nUS-
  n	X9   X8   /n
XI   XH   /n[        R                  " U[        R                  " X*U5      5      nXg4$ )Nr0   )r   r5   appendargmaxinterp)rL   	y_predictmax_fprfprtpr_new_fprnew_tpridx_outidx_inx_interpy_interps               rM   _partial_roc,_partial_roc_auc_score.<locals>._partial_roc   s    2!Wn%))G-Wn%))CM*1K.K.))GRYYw(%KL!!rO   r         ?r0   )r	   )rL   ru   rv   r   rz   r{   partial_aucfpr1fpr2min_areamax_areas              rM   _partial_roc_auc_scorer      sc    

" $Fw?Gg'K DDdk"dk2H{H!{-(2EFFGGrO   dropTc                    [        SS9u  pn[        X5      n[        XU S9u  pVn[        XV5      n[	        XSS9  [        U[        X5      5        UR                  UR                  :X  d   eUR                  UR                  :X  d   eg )NTrA   drop_intermediater+   decimal)rN   r\   r   r	   r#   r"   r   r4   )	r   rL   ry   rJ   expected_aucrw   rx   
thresholdsroc_aucs	            rM   test_roc_curver      s~     )5Fw(L$VMCj#mGgQ?v!?@99		!!!99
(((((rO   c                  Z   [         R                  R                  S5      n [         R                  " S/S-  S/S-  -   5      nU R	                  SSS9n[        XSS9u  p4nUS   S:X  d   eUS	   S:X  d   eUR                  UR                  :X  d   eUR                  UR                  :X  d   eg )
Nr   2   r0      d   sizeTr   rQ   )r5   r9   r:   arrayrandintr   r4   )rG   rL   rK   rw   rx   thrs         rM   test_roc_curve_end_pointsr      s     ))


"CXXqcBh!r)*F[[[%FfEMCcq6Q;;r7a<<99		!!!99		!!!rO   c                  b   [        SS9u  pn[        X5      u  p4n/ nU HK  n[        R                  " X':  U -  5      n[        R                  " U 5      n	UR	                  SU-  U	-  5        MM     [        XFSS9  UR                  UR                  :X  d   eUR                  UR                  :X  d   eg )NTr   r^   r+   r   )rN   r   r5   rT   rr   r#   r4   )
rL   ry   rJ   rw   rx   r   tpr_correctttprF   s
             rM   test_roc_returns_consistencyr      s     )5Fw$V5Cj KVVW\V+,FF6N38a<(  c:99		!!!99
(((((rO   c                      [        SS9u  pn[        R                  " [        5         [	        X5        S S S 5        g ! , (       d  f       g = f)NFr   )rN   pytestraises
ValueErrorr   )rL   ry   rJ   s      rM   test_roc_curve_multir      s1    (6Fw	z	"&" 
#	"	"s	   <
A
c                      [        SS9u  pn[        XS-
  5      u  p4n[        X45      n[        USSS9  UR                  UR                  :X  d   eUR                  UR                  :X  d   eg )NTr   r   ?r+   r   )rN   r   r	   r#   r4   )rL   ry   rJ   rw   rx   r   r   s          rM   test_roc_curve_confidencer     sg    (5Fw$Vs];Cj#mGgtQ799		!!!99
(((((rO   c                     [        SS9u  pn[        R                  " U R                  5      n[	        X5      u  pEn[        XE5      n[        USSS9  UR                  UR                  :X  d   eUR                  UR                  :X  d   e[        R                  " U R                  5      n[	        X5      u  pEn[        XE5      n[        USSS9  UR                  UR                  :X  d   eUR                  UR                  :X  d   e[	        X5      u  pEn[        XE5      n[        USSS9  UR                  UR                  :X  d   eUR                  UR                  :X  d   eg )NTr   r   r+   r   g(\?)rN   r5   onesr4   r   r	   r#   zeros)rL   predrJ   trivial_predrw   rx   r   r   s           rM   test_roc_curve_hardr     s1   +48F' 776<<(L$V:Cj#mGgtQ799		!!!99
((((( 88FLL)L$V:Cj#mGgtQ799		!!!99
((((( %V2Cj#mGgtQ799		!!!99
(((((rO   c                     / SQn / SQnSn[         R                  " [        US9   [        X5      u  p4nS S S 5        [	        W[
        R                  " [        W5      [
        R                  5      5        UR                  WR                  :X  d   eUR                  UR                  :X  d   eSn[         R                  " [        US9   [        U  Vs/ s H  nSU-
  PM
     snU5      u  p4nS S S 5        [	        U[
        R                  " [        U5      [
        R                  5      5        UR                  UR                  :X  d   eUR                  UR                  :X  d   eg ! , (       d  f       GN7= fs  snf ! , (       d  f       N= f)N)
r0   r0   r0   r0   r0   r0   r0   r0   r0   r0   )
r   r0   r   r0   r   r0   r   r0   r   r0   INo negative samples in y_true, false positive value should be meaninglessmatchHNo positive samples in y_true, true positive value should be meaninglessr0   )
r   warnsr   r   r$   r5   fullrV   nanr4   )rL   rK   expected_messagerw   rx   r   xs          rM   test_roc_curve_one_labelr   +  s8   +F+F 	T  
,4D	E(8* 
F sBGGC
ORVV<=99		!!!99
((((( 	S  
,4D	E()@A!a%)@&I* 
F sBGGC
ORVV<=99		!!!99
(((((# 
F	E *A 
F	Es)   E#
E:E5
E:#
E25E::
Fc                  2   SS/n SS/n[        X5      u  p#n[        X5      n[        U/ SQ5        [        U/ SQ5        [        US5        SS/n SS/n[        X5      u  p#n[        X5      n[        U/ SQ5        [        U/ SQ5        [        US5        SS/n SS/n[        X5      u  p#n[        X5      n[        USS/5        [        USS/5        [        US5        SS/n SS/n[        X5      u  p#n[        X5      n[        U/ SQ5        [        U/ SQ5        [        US5        SS/n SS/n[        X5      u  p#n[        X5      n[        USS/5        [        USS/5        [        US5        SS/n SS	/nS
n[        R
                  " [        US9   [        X5      u  p#nS S S 5        [        U/ SQ5        [        U[        R                  [        R                  [        R                  /5        Sn[        R
                  " [        US9   [        X5      nS S S 5        [        R                  " W5      (       d   eSS/n SS	/nSn[        R
                  " [        US9   [        X5      u  p#nS S S 5        [        U[        R                  [        R                  [        R                  /5        [        U/ SQ5        Sn[        R
                  " [        US9   [        X5      nS S S 5        [        R                  " U5      (       d   e[        R                  " SS/SS//5      n [        R                  " SS/SS//5      n[        R
                  " [        US9   [        XSS9  S S S 5        [        R
                  " [        US9   [        XSS9  S S S 5        [        [        XSS9S5        [        [        XSS9S5        [        R                  " SS/SS//5      n [        R                  " SS/SS//5      n[        R
                  " [        US9   [        XSS9  S S S 5        [        R
                  " [        US9   [        XSS9  S S S 5        [        [        XSS9S5        [        [        XSS9S5        [        R                  " SS/SS//5      n [        R                  " SS/SS//5      n[        [        XSS9S5        [        [        XSS9S5        [        [        XSS9S5        [        [        XSS9S5        [        R                  " SS/SS//5      n [        R                  " SS/SS//5      n[        [        XSS9S5        [        [        XSS9S5        [        [        XSS9S5        [        [        XSS9S5        g ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNG= f! , (       d  f       GN= f! , (       d  f       GNt= f! , (       d  f       GNZ= f! , (       d  f       GN= f! , (       d  f       GN= f)Nr   r0   r   r   r0   r   r0   r0   r^           r         ?      ?r   r   r   r   r^   OOnly one class is present in y_true. ROC AUC score is not defined in that case.r   macroaverageweightedsamplesmicro)r   r   r#   r"   r   r   r   r5   r   mathisnanr   )rL   rJ   rx   rw   ry   r   r   r	   s           rM   test_roc_curve_toydatar   F  s?   VF!fGF,KCaF,Gc9-c9-%VF!fGF,KCaF,Gc9-c9-%VF!fGF,KCaF,GcAq6*cAq6*%VF!fGF,KCaF,Gc9-c9-%VFCjGF,KCaF,GcAq6*cAq6*% VFTlG 	S  
,4D	E0! 
Fc?3cBFFBFFBFF#;<	5  
,4D	EF, 
F::c??? VFTlG 	T  
,4D	E0! 
FcBFFBFFBFF#;<c?3	5  
,4D	EF, 
F::c??? XX1v1v&'FhhAA'(G	,4D	Efw7 
F	,4D	Efz: 
FfyI3OfwGMXX1v1v&'FhhAA'(G	,4D	Efw7 
F	,4D	Efz: 
FfyI3OfwGMXX1v1v&'FhhAA'(GfwGKfzJANfyI1MfwGKXX1v1v&'Fhhc
S#J/0GfwGMfzJCPfyI3OfwGM} 
F	E 
F	E 
F	E 
F	E 
F	E	E	E 
F	E	E	Es`   V	
VV-V?WW#W5=X	
V
V*-
V<?
W
W #
W25
X
Xc            	          / SQn / SQn[        XSS9u  p#n[        U[        R                  SSS/5        / SQn / S	Qn[        XSS9u  p#n[        U[        R                  SS
SSS/5        g )Nr   r   r   r   r0   r0   r   皙?r   333333?ffffff?r^   Tr   r^   r   r   r   r   r   r   r   r   r   r0   r0   r0   r0   r0   r0   r   皙?r   r   r   皙?r   r   r   r   r   r   r^   r   r   )r   r#   r5   inf)rL   rJ   rx   rw   r   s        rM    test_roc_curve_drop_intermediater     sl    F,G$VMCjj2663S*AB 5FOG$VMCjj2663S#s*KLrO   c                     / SQn / SQn[         R                  " SS5      n[        XUS9u  p4n[         R                  " U5      S:  R	                  5       S:X  d   e[         R                  " U5      S:  R	                  5       S:X  d   eg )Nr   r   r0   r0   r0   )r   r   333333?皙?r   r      sample_weightr   )r5   repeatr   diffrT   )rL   rJ   r   rw   rx   ry   s         rM   !test_roc_curve_fpr_tpr_increasingr     sv     F'GIIc1%MF=IKCaGGCL1!!#q(((GGCL1!!#q(((rO   c                  &   SS/n SS/n[        [        X5      S5        SS/n SS/n[        [        X5      S5        / SQn / SQn[        [        X5      S5        SS/n SS/n[        [        X5      S5        / SQn / SQn[        [        X5      S5        g )Nr   r0   r   r0   r   r   r   r   r   r0   )r#   r	   )r   rC   s     rM   test_aucr     s    	
AA	
AAc!i-	
AA	
AAc!i-AAc!i-	
AA	
AAc!i+AAc!i-rO   c                     [         R                  " [        5         [        / SQSS/5        S S S 5        [         R                  " [        5         [        S/S/5        S S S 5        / SQn / SQnSR	                  [
        R                  " U 5      5      n[         R                  " [        [        R                  " U5      S9   [        X5        S S S 5        g ! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g = f)	Nr   r   r   r   )r+   r0   r      )r            z+x is neither increasing nor decreasing : {}r   )	r   r   r   r	   formatr5   r   reescape)r   rC   error_messages      rM   test_auc_errorsr     s    	z	"Oc3Z( 
# 
z	"SEC5 
# 	AAAHHRSUM	z=)A	BA	 
C	B 
#	" 
#	" 
C	Bs#   CC%?C6
C"%
C36
Dzy_true, labels)r   r0   r   r+   r   r0   r+   )abr   c)r   r   r   c           
      V   [         R                  " / SQ/ SQ/ SQ/ SQ/5      n[        / SQ/ SQ5      n[        / SQ/ SQ5      nX4-   S	-  n[        / S
Q/ SQ5      n[        / SQ/ SQ5      nXg-   S	-  n[        SS/SS/5      n	[        SS/SS/5      n
X-   S	-  nXX-   U-   S-  n[        [        XUSS9U5        XXU/n/ SQn[         R                  " XS9n[        [        XUSSS9U5        Sn[
        R                  " [        US9   [        XUSS S9  S S S 5        g ! , (       d  f       g = f)Nr   r   r   r   r   r   ffffff?r   333333?r   r   r   r0   r   r0   )r   r   r   r   r0   r   )r   r   r   r+   r0   r0   r   )r   r   r   r   )r   r   r   r0   r   r   r   r   r   r   ovolabelsmulti_class)r   r   r   )weightsr   r  r  r   z6average=None is not implemented for multi_class='ovo'.r   )r5   r   r   r"   r   r   r   NotImplementedError)rL   r  y_scoresscore_01score_10average_score_01score_02score_20average_score_02score_12score_21average_score_12ovo_unweighted_scorepair_scores
prevalenceovo_weighted_scorer   s                    rM   #test_multiclass_ovo_roc_auc_toydatar    sI    xx	/+<mLH Y(89HY8H +q0 Y7HY(89H +q0 aVc3Z0HaVc3Z0H +q0 -?BRRVWWfv5I $7GHK#JKDV
	
 		 MM	*-	@fv5RVW 
A	@	@s   D
D()r   r+   r   r+   )r   dr   r  )r   r   r  c           
          [         R                  " / SQ/ SQ/ SQ/ SQ/5      n[        / SQ/ SQ5      n[        / SQ/ SQ5      nX4-   S	-  n[        [        XUS
S9U5        [        [        XUS
SS9U5        g )N)r   r   r   )r   r   r   )皙?r   ?)r   r   r   )r0   r   r0   r   )r   r   r  r   r   r0   r   r0   )r   r   r  r   r+   r   r   r   r  r5   r   r   r"   )rL   r  r  r  r  	ovo_scores         rM   *test_multiclass_ovo_roc_auc_toydata_binaryr  1  s~     xx	/+<oNH \+@AH\+@AH$)Ifv5I9
 V
	
 		rO   )r   r0   r+   r+   )r   r   r   r   c           
         [         R                  " / SQ/ SQ/ SQ/ SQ/5      n[        / SQUS S 2S4   5      n[        / SQUS S 2S4   5      n[        / S	QUS S 2S
4   5      n[        [        XSUS S9X4U/5        X4-   U-   S-  n[        [        XSUS9U5        US-  US-  -   US-  -   n[        [        XSUSS9U5        g )Nr^   r   r   )r   r   r   )r   r   r   )r   r   r   r0   r   r   r   r   )r   r0   r   r   r0   r   r   r0   r0   r+   ovr)r  r  r   g      @)r  r  r   r   r   r  )rL   r  r  out_0out_1out_2result_unweightedresult_weighteds           rM   #test_multiclass_ovr_roc_auc_toydatar'  U  s     xx	/?OLH
 ,A7E,A7E,A7EfE&RVW	u .#5fE&I dlUT\1ECK?O%
	
 		rO   zmulti_class, average))r!  r   )r!  r   )r   r   c           	      .   [         R                  " / SQ5      n/ SQ/ SQ/ SQ/ SQ/n[        [        X#XS9S5        / SQ/ SQ/ SQ/ SQ/n[        X$XS9S:  d   eS[         R                  " S	5      -  n[        X%XS9[
        R                  " S
5      :X  d   eg )N)r   r0   r+   r   )r   r   r   r^   )r   r^   r   r   )r   r   r^   r   )r   皙?r)  r   r  r   r^   r   )r   r   r   )r5   r   r"   r   r   r   approx)r  r   rL   	y_perfecty_imperfecty_chances         rM   0test_perfect_imperfect_chance_multiclass_roc_aucr/    s     XXl#F 	 	I f[R 		K 	f{T
		
 bggfo%Hk	s	  rO   c                    U n[         R                  R                  / SQSUS9n[        R                  " U Vs/ s H/  n[         R
                  R                  SX1S9R                  5       PM1     sn5      n[        U/ SQS9n[        UR                  5       UR                  5       5      u  pgn[        Xg5      n	[        XBSS	S
9n
U	[        R                  " U
5      :X  d   eg s  snf )N)       @r^   r     )r   r/   r0   )nrF   r/   r   )classesr!  r   r*  )r   	dirichletrvsr5   asarraymultinomialrs   r   r   ravelr	   r   r   r+  )global_random_seedseedrK   y_pred_irL   y_onehotrw   rx   ry   roc_auc_by_handroc_auc_autos              rM   test_micro_averaged_ovr_roc_aucr@    s    D
 __  t$ OFZZ #	
" !!A!ELLN"	
F fi8HHNN,flln=KCa#mO UGTLfmmL9999	
s   6Czmsg, y_true, labelsz!Parameter 'labels' must be uniquer   r+   r   )r   r   r   zKNumber of classes in y_true not equal to the number of columns in 'y_score'z"Parameter 'labels' must be ordered)r   r   r   zMNumber of given labels, 2, not equal to the number of columns in 'y_score', 3r0   r   r   zMNumber of given labels, 4, not equal to the number of columns in 'y_score', 3)r   r0   r+   r   )r   r   r   r  z2'y_true' contains labels not in parameter 'labels')r   r   r   er  r   r!  c                     [         R                  " / SQ/ SQ/ SQ/ SQ/5      n[        R                  " [        U S9   [        XX#S9  S S S 5        g ! , (       d  f       g = f)Nr   r   r   r   r   r   )r5   r   r   r   r   r   )msgrL   r  r  r  s        rM   *test_roc_auc_score_multiclass_labels_errorrE    sF    R xx	/+<mLH 
z	-fvO 
.	-	-s   A
Azmsg, kwargszLaverage must be one of \('macro', 'weighted', None\) for multiclass problemsr   )r   r  zUaverage must be one of \('micro', 'macro', 'weighted', None\) for multiclass problemszksample_weight is not supported for multiclass one-vs-one ROC AUC, 'sample_weight' must be None in this case)r  r   z|Partial AUC computation not available in multiclass setting, 'max_fpr' must be set to `None`, received `max_fpr=0.5` insteadr   )r  rv   z'multi_class must be in \('ovo', 'ovr'\)c                     [        S5      nUR                  SS5      n[        U5      nUR                  SSSS9n[        R
                  " [        U S9   [        XT40 UD6  S S S 5        g ! , (       d  f       g = f)N     r   r   r   r   )r)   randr%   r   r   r   r   r   )rD  kwargsrG   rJ   y_probrL   s         rM   #test_roc_auc_score_multiclass_errorrL    sc    N S
!Chhr1oGWF[[AB['F	z	-f// 
.	-	-s   A))
A7c                  4   [        S5      n U R                  S5      n[        R                  " SSS9nSn[        R
                  " [        US9   [        X!5        S S S 5        [        R                  " SSS9n[        R
                  " [        US9   [        X!5        S S S 5        [        R                  " SSSS9n[        R
                  " [        US9   [        X!5        S S S 5        g ! , (       d  f       N= f! , (       d  f       Nc= f! , (       d  f       g = f)NrG  
   r8   dtyper   r   rQ   )
r)   rI  r5   r   r   r   r   r   r   r   )rG   rK   rL   warn_messages       rM   test_auc_score_non_binary_classrR  >  s     S
!CXXb\FXXb&F	5  
,L	Af% 
BWWRu%F	,L	Af% 
BWWR5)F	,L	Af% 
B	A 
B	A 
B	A 
B	As$   C'C8D	'
C58
D	
Dc                     [         R                  " / SQ5      n [         R                  " SS/5      n[        R                  " [        SS9   [        X5        SSS5        [         R                  " / SQ5      n [         R                  " S[         R                  SS	/5      n[        R                  " [        S
S9   [        X5        SSS5        g! , (       d  f       N|= f! , (       d  f       g= f)zKTest `_sort_inputs_and_compute_classification_thresholds` input validation.r   r   r   inconsistent numbers of samplesr   Nr  r   r   zInput.*contains NaN)r5   r   r   r   r   r   r   rL   rJ   s     rM   Gtest_sort_inputs_and_compute_classification_thresholds_input_validationrV  S  s     XXi FhhSz"G	z)J	K:6K 
L XXl#FhhRVVS#./G	z)>	?:6K 
@	? 
L	K 
@	?s   
C7C
C
C+c                  p   [         R                  " / SQ5      n [         R                  " / SQ5      n[         R                  " / SQ5      n[        XU5      u  p4pV[        U5      [        U5      s=:X  a  [        U5      s=:X  a  S:X  d   e   eSU;  d   eSU;  d   e[        X5      u    pgnUb   e[         R                  " / SQ5      n [         R                  " / S	Q5      n[         R                  " / S
Q5      n[        R
                  " [        SS9   [        XU5        SSS5        g! , (       d  f       g= f)zJTest zero weights in `_sort_inputs_and_compute_classification_thresholds`.r   r0   r   r0   r   r0   )r   r   r   r   r   r   )r   r1  r^         ?r   r   r   r   r   Nr   )r   r   r   )r   r   r   z$Sample weights must contain at leastr   )r5   r   r   rV   r   r   r   )rL   rJ   r   y_true_sortedy_score_sortedweight_sortedry   weights           rM   Ctest_sort_inputs_and_compute_classification_thresholds_zero_weightsr^  d  s	   XX()Fhh56GHH;<M 	;]	
 4M= }^!4OM8JOaOOOOOn$$$n$$$ IOAq! >> XXi Fhh'GHH_-M	z)O	P:]	
 
Q	P	Ps   D''
D5c                  Z   [         R                  " / SQ5      n [         R                  " / SQ5      n[         R                  " / SQ5      n[        XU5      u  p4pV[         R                  " USS USS :  5      (       d   e[	        U[         R                  " / SQ5      5        [	        U[         R                  " / SQ5      5        [	        US	S
 SS/5        [	        US	S
 SS/5        [         R                  " / SQ5      n[        X5      u      p[	        U[         R                  " S
/5      5        g)zETest sorting in `_sort_inputs_and_compute_classification_thresholds`.)r   r0   r   r0   r0   )r   r   r   r   r   )r^   r1  rY  r   r   NrQ   r0   )r1  r   rY  r   r^   )r   r0   r   r   r+   r   r   r   )r   r   r   r   r   )r5   r   r   allr$   )	rL   rJ   r   rZ  r[  r\  threshold_idxsy_score_samery   s	            rM   >test_sort_inputs_and_compute_classification_thresholds_sortingrc    s    XXo&Fhh01GHH67M 	;]	
 AM= 66."%);;<<<<}bhh/H&IJ~rxx'=>~a*S#J7}Qq)Aq62 8856LPAq! ~rxx}5rO   c                    [         R                  R                  U 5      nSnUR                  SSSS9nUR	                  SS9n[         R
                  " U5      nX%-
  n[        X45      u  pxpn[        U5      [        U5      s=:X  a0  [        U	5      s=:X  a   [        U
5      s=:X  a  [        U5      :X  d   e   e[        X-   U5        [        Xx-   U5        [        Xx-   U	-   U
-   U5        g)z.Smoke test for confusion_matrix_at_thresholds.r   r   r+   r   N)	r5   r9   r:   r   uniformrT   r   rV   r!   )r:  rG   rD   rL   rJ   ra   n_negtnsfpsfnstpsr   s               rM   #test_confusion_matrix_at_thresholdsrk    s    
))

 2
3CI[[AC[(Fkksk#GFF6NEE%CF%T"Cc
s8s3xJ3s8Js3xJ3z?JJJJJCIu%CIu%CIOc)95rO   
curve_funcc                     [        S5      nUR                  SSSS9nUR                  S5      nSn[        R                  " [
        US9   U " X#5        S S S 5        g ! , (       d  f       g = f)NrG  r   r   rN  r   z"multiclass format is not supportedr   )r)   r   rI  r   r   r   )rl  rG   rL   rK   rD  s        rM   4test_confusion_matrix_at_thresholds_multiclass_errorrn    sT    
S
!C[[AB['FXXb\F
.C	z	-6" 
.	-	-s   		A
A)c                    Sn[         R                  " [        US9   U " [        R                  " SS/SS9SS/5        S S S 5        [         R                  " [        US9   U " [        R                  " SS/[
        S9SS/5        S S S 5        / S	QnU " / S
QU5      nU " / SQU5      n[        X45       H$  u  pV[        R                  R                  XV5        M&     g ! , (       d  f       N= f! , (       d  f       Nm= f)Nzy_true takes value in {'a', 'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.r   r   r   z<U1rO  r   r^   )r   r^   r   gzG?r   r0   r0   r   )r   r^   r^   r   )	r   r   r   r5   r   objectziptestingr!   )rl  rD  rK   	int_curvefloat_curveint_curve_partfloat_curve_parts          rM   6test_confusion_matrix_at_thresholds_implicit_pos_labelrx    s    
	  
z	-288S#Je4sCjA 
. 
z	-288S#Jf5SzB 
.
 #F<0I16:K,/	,G(


"">D -H 
.	- 
.	-s   !C%C*
C'*
C8z.ignore:Support for labels represented as byteslabels_typeri   r   c                     [        SS/U5      nSn[        R                  " [        US9   U " USS/5        S S S 5        g ! , (       d  f       g = f)N   a   bz8Support for labels represented as bytes is not supportedr   r   r^   )r    r   r   	TypeError)rl  ry  r  rD  s       rM   <test_confusion_matrix_at_thresholds_implicit_bytes_pos_labelr~    sA      tk:F
DC	y	,6C:& 
-	,	,s	   ?
Ac                     / SQn/ SQn/ SQnU " XUS9nU " US S US S US S S9n[        XE5       H  u  pg[        Xg5        M     g )Nr   r   r   r   r   r   )r0   r0   r0   r   r   r   rQ   )rr  r!   )rl  rL   rJ   r   result_1result_2arr_1arr_2s           rM   6test_confusion_matrix_at_thresholds_zero_sample_weightr    sY    F'G%M&GH&"+ws|=QTRTCUVHH/% 0rO   c                    [        SS9u  pn[        XU 5        [        USS  USS  U S9u  pEnUS   S:X  d   eUS   USS  R                  5       :X  d   eSU[        R
                  " US:H  5      '   UR                  5       n[        XU 5        [        Xq5        / SQn/ S	Qn	[        XU S9u  pEnU (       a+  [        U/ S
Q5        [        U/ SQ5        [        U/ SQ5        O*[        U/ SQ5        [        U/ SQ5        [        U/ S	Q5        UR                  UR                  :X  d   eUR                  UR                  S-   :X  d   eg )NTr   r0   r   r   r^   rQ   r0   r   r   r0   )r0   r+   r   r   )r   QUU?r^   r^   )r^   r   r   r   )r0   r+   r   )r   r  r   r^   r^   )r^   r   r   r   r   )
rN   _test_precision_recall_curver   meanr5   wherecopyr$   r!   r   )
r   rL   ry   rJ   rF   rr   y_true_copyr  predict_probass
             rM   test_precision_recall_curver    s.   (5Fw $7 %VABZPTUGA!Q43;;Q46!":??$$$$ %'F288FaK !++-K $7{+F!N$VtTGA!67/09%;<45<(66QVV66QVVaZrO   c                    [        XUS9u  p4n[        X5      n[        USS5        [        U[        X5      5        [	        [        X5      USS9  UR                  UR                  :X  d   eUR                  UR                  S-   :X  d   e[        U [        R                  " U5      US9u  p4nUR                  UR                  :X  d   eUR                  UR                  S-   :X  d   eg )Nr   g+?r   r+   r   r0   )	r   ro   r#   r
   r"   rg   r   r5   
zeros_like)rL   rJ   r   rF   r  r   precision_recall_aucs          rM   r  r    s    -fQUVA*26C2E1=5fF 6+-A1 66QVV66Z__q((((-g&$A* 66QVV66Z__q((((rO   c           
         [         R                  " SS9   SS/nSS/n[        XU S9u  p4n[        X5      n[	        U/ SQ5        [	        U/ SQ5        [        US5        SS/nSS/n[        XU S9u  p4n[        X5      n[	        U/ S	Q5        [	        U/ S
Q5        [        US5        SS/nSS/n[        XU S9u  p4n[        X5      n[	        USS/5        [	        USS/5        [        US5        SS/nSS/n[        XU S9u  p4n[        X5      n[	        U/ SQ5        [	        U/ SQ5        [        US5        SS/nSS/n[        XU S9u  p4n[        X5      n[	        USS/5        [	        USS/5        [        US5        SS/nSS/n[        R                  " [        SS9   [        XU S9u  p4nS S S 5        [        R                  " [        SS9   [        X5      nS S S 5        [        U/ SQ5        [        U/ SQ5        [        US5        SS/nSS/n[        XU S9u  p4n[        [        X5      S5        [	        U/ SQ5        [	        U/ SQ5        [         R                  " SS/SS//5      n[         R                  " SS/SS//5      n[        R                  " [        SS9   [        [        XSS9S5        S S S 5        [        R                  " [        SS9   [        [        XSS9S5        S S S 5        [        [        XSS9S5        [        [        XSS9S5        [         R                  " SS/SS//5      n[         R                  " SS/SS//5      n[        R                  " [        SS9   [        [        XSS9S5        S S S 5        [        R                  " [        SS9   [        [        XSS9S5        S S S 5        [        [        XSS9S5        [        [        XSS9S5        [         R                  " SS/SS//5      n[         R                  " SS/SS//5      n[        [        XSS9S5        [        [        XSS9S5        [        [        XSS9S5        [        [        XSS9S5        [         R                  " SS/SS//5      n[         R                  " SS/SS//5      n[        R                  " [        SS9   [        [        XSS9S5        S S S 5        [        [        XSS9S5        [        R                  " [        SS9   [        [        XSS9S5        S S S 5        [        R                  " [        SS9   [        [        XSS9S5        S S S 5        [         R                  " SS/SS//5      n[         R                  " SS/SS//5      n[        [        XSS9S5        [        [        XSS9S5        [        [        XSS9S5        [        [        XSS9S5        [         R                  " SS/SS//5      n[         R                  " SS/SS//5      n[        [        XSS9S5        [        [        XSS9S5        [        [        XSS9S5        [        [        XSS9S5        S S S 5        [         R                  " SS9   [         R                  " SS/SS//5      n[         R                  " SS/SS//5      n[        R                  " [        SS9   [        [        XSS9S5        S S S 5        S S S 5        g ! , (       d  f       GN\= f! , (       d  f       GNA= f! , (       d  f       GNw= f! , (       d  f       GNS= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNg= f! , (       d  f       GNC= f! , (       d  f       GN== f! , (       d  f       N= f! , (       d  f       g = f)Nraise)r`  r   r0   r   )r   r0   r0   r   r^   )r   r   r^   r  r   r   r   r   z!No positive class found in y_truer   r   )r^   r^   r^   )r0   r   r   r   r   r   r   r   ignore)r5   errstater   r
   r#   r"   r   r   UserWarningr!   r   )r   rL   rJ   rF   r  ry   auc_prcs          rM   #test_precision_recall_curve_toydatar  )  s   		!Qa&(DQa)&:!![1!!Y/GS)Qa&(DQa)&:!!_5!!_5 	GS)Qa&(DQa)&:!!c1X.!!c1X.GS)Qa&(DQa)&:!![1!!Y/GS)Q*(DQa)&:!!c1X.!!aX.GS)Q,\\+-PQ,VPTUGA! R\\+-PQ-f>G R9%9%#Q,(DQa3FDcJ!!_5!!]3 Aq6Aq6*+((QFQF+,\\+-PQ'I3 R \\+-PQ'Lc R 	#FYG	
 	/QSVWAq6Aq6*+((QFQF+,\\+-PQ'I3 R \\+-PQ'Lc R 	#FYG	
 	/QSVWAq6Aq6*+((QFQF+,#FWEs	
 	#FZH#	
 	#FYG	
 	#FWEs	
 Aq6Aq6*+((QFQF+,\\+-PQ'I3 R 	#FZH#	
 \\+-PQ'KS R \\+-PQ'I3 R
 Aq6Aq6*+((QFQF+,/QSVW#FZH#	
 	#FYG	
 	/QSVWAq6Aq6*+((S#Jc
34#FWEs	
 	#FZH#	
 	#FYG	
 	#FWEs	
i 
"p 
	"Aq6Aq6*+((QFQF+,\\+-PQ'La R	 
#	"S RQQQ  RQ RQ RQ RQ4 RQ RQ RQs 
"	!x RQ	 
#	"s  F
^$\-!^$\C^$!\&6!^$\8,B^$-]
!^$#]8D	^$].5^$^  !^$^D^$
A_^61_
\	^$
\#	^$&
\5	0^$8
]	^$

]	^$
]+	&^$.
]=	8^$ 
^	
^$
^!	^$$
^36
_	 _
_c                     / SQn / SQn[        XSS9u  p#n[        U/ SQ5        / SQn / SQn[        XSS9u  p#n[        U/ SQ5        / S	Qn / S
Qn[        XSS9u  p#n[        USS/5        / SQn / S
Qn[        XSS9u  p#n[        U/ S
Q5        g)z9Check the behaviour of the `drop_intermediate` parameter.r   r   Tr   )r   r   r^   r   r   )r   r   r   r   r   r^   r   r   r   r   )r   r   r   r   r   r   r0   r0   r0   r0   N)r   r!   )rL   rJ   rk   rl   r   s        rM   -test_precision_recall_curve_drop_intermediater    s    F,G$:4%!Iz J0 5FOG$:4%!Iz J >? F"G$:4%!Iz Jc
+ F"G$:4%!Iz J 45rO   c                      [         R                  " S[        S9n SU S S S2'   [         R                  " S5      n[	        X5      S:X  d   eg )Nr   rO  r0   r   r   )r5   r   r8   r   r
   rU  s     rM   &test_average_precision_constant_valuesr    sC    
 XXc%FF3Q3KggclG #63t;;;rO   c                  
   [         R                  " SS/5      n [         R                  " SS/5      n[        R                  " S5      n[        R
                  " [        US9   [        XSS9  S S S 5        g ! , (       d  f       g = f)Nr   r0   z;pos_label=2 is not a valid label. It should be one of [0 1]r   r+   rW   )r5   r   r   r   r   r   r   r
   rL   rK   err_msgs      rM   4test_average_precision_score_binary_pos_label_errorsr    sY    XXq!fFXXq!fFiiUVG	z	1!< 
2	1	1    A44
Bc                  
   [         R                  " SS/SS/SS/SS//5      n [         R                  " SS/SS/SS/SS//5      nSn[        R                  " [        US9   [        XSS	9  S S S 5        g ! , (       d  f       g = f)
Nr0   r   r   r   r   r   znParameter pos_label is fixed to 1 for multilabel-indicator y_true. Do not set pos_label or set pos_label to 1.r   r  r5   r   r   r   r   r
   r  s      rM   8test_average_precision_score_multilabel_pos_label_errorsr    s     XX1v1v1v1v67FXXSzC:SzC:FGF	6  
z	1!< 
2	1	1r  c            	         [         R                  " / SQ5      n [         R                  " / SQ/ SQ/ SQ/ SQ/ SQ/ SQ/5      nSn[        R                  " [        US9   [        XSS	9  S S S 5        g ! , (       d  f       g = f)
Nr   r0   r+   r   r0   r+   )r   r   r   )r   r   r   )r   r   r   )r   r   r   zdParameter pos_label is fixed to 1 for multiclass y_true. Do not set pos_label or set pos_label to 1.r   r   r  r  )rL   rJ   r  s      rM   8test_average_precision_score_multiclass_pos_label_errorsr    sf    XX()Fhh	
	G	6  
z	11= 
2	1	1s   A00
A>c                     [         R                  " / SQ5      n [         R                  " / SQ5      n[        R                  " S5      n[        R
                  " [        US9   [        X5        SSS5        [        R
                  " [        US9   [        XSS9  SSS5        [        R
                  " [        US9   [        X5        SSS5        g! , (       d  f       Nh= f! , (       d  f       NM= f! , (       d  f       g= f)zNTest ranking metrics, with multiclass support, raise if shape `y_score` is 1D.r  )r   r   r   r   r   r   z7`y_score` needs to be of shape `(n_samples, n_classes)`r   Nr!  )r  )
r5   r   r   r   r   r   r   r
   r   r   )rL   rJ   rD  s      rM   Dtest_multiclass_ranking_metrics_raise_for_incorrect_shape_of_y_scorer  (  s    XX()Fhh56G
))M
NC	z	-0 
.	z	-f59 
.	z	-V- 
.	-	 
.	-	-	-	-	-s$    CC9C0
C
C-0
C>c                  L   [        SS9u  pn[        X5      n[        U SU-  5      n[        U SU-  5      n[        XS-
  5      nX4:X  d   eX5:X  d   eX6:X  d   e[        X5      n[        U SU-  5      n[        U SU-  5      n	[        XS-
  5      n
Xx:X  d   eXy:X  d   eXz:X  d   eg )NTr   r   gư>rN  )rN   r   r
   )rL   ry   rJ   r   roc_auc_scaled_uproc_auc_scaled_downroc_auc_shiftedpr_aucpr_auc_scaled_uppr_auc_scaled_downpr_auc_shifteds              rM   test_score_scale_invariancer  6  s     )5FwF,G%fcGm<'w?#FbL9O''')))%%%$V5F.vsW}E0H,Vr\BN%%%'''###rO   z(y_true,y_score,expected_fpr,expected_fnrr   r   )r   r   r   )r   r   r0   r   r   r   r   )r   r   r   r   r   r   )r0   r0   r   r   )r   r0   r0   r0   r   r   )r   r   r   c                 L    [        X5      u  pEn[        XB5        [        XS5        g Nr   r!   rL   rJ   expected_fprexpected_fnrrw   fnrry   s          rM   test_det_curve_toydatar  O  s"    6 F,KCaC&C&rO   )rL   rJ   r  r  r   )r0   r0   r   r   )r0   r0   r   c                 J    [        XUS9u  pVn[        XR5        [        Xc5        g )Nr   r  )rL   rJ   r  r  r   rw   r  ry   s           rM    test_det_curve_drop_intermediater  p  s%    & F?PQKCaC&C&rO   r   r   r   c                 L    [        X5      u  pEn[        XB5        [        XS5        g r  r  r  s          rM   test_det_curve_tie_handlingr    s"     F,KCaC&C&rO   c                  T    [        [        / SQ/ SQ5      [        / SQ/ SQ5      5        g )Nr   r   r   )r   r   r   r   r0   r0   )r!   r    rO   rM   test_det_curve_sanity_checkr    s     )[)$&<=rO   rJ   )r   r   r   r   r0   c                     [        / SQ[        R                  " SU 5      S9u  pn[        USS/5        [        USS/5        [        X0[        R                  /5        g )NrX  r   rU  r0   r   )r   r5   r   r!   r   )rJ   rw   r  rm   s       rM   test_det_curve_constant_scoresr    sP    #!2771g+>Ci C!Q C!Q I01rO   rL   )r   r   r   r   r   r0   r   )r   r   r   r0   r0   r0   )r   r   r0   r0   r0   r0   )r   r0   r0   r0   r0   r0   c                 P    [        X S9u  pn[        US/5        [        US/5        g )NrU  r   r  )rL   rw   r  ry   s       rM   test_det_curve_perfect_scoresr    s*     6:KCaC!C!rO   zy_true, y_pred, err_msgrT  r   r   r   z#Only one class is present in y_truer0   r0   r0   )cancerr  
not cancer)r   r   r   zpos_label is not specifiedc                     [         R                  " [        US9   [        X5        S S S 5        g ! , (       d  f       g = f)Nr   )r   r   r   r   r  s      rM   test_det_curve_bad_inputr    s&      
z	1&! 
2	1	1   /
=c                  P   S/S-  S/S-  -   n [         R                  " / SQ5      nSU-
  n[        U USS9u  p4n[        U USS9u  pgnUS   [        R                  " S	5      :X  d   eUS   [        R                  " S
5      :X  d   e[        X7S S S2   5        [        XFS S S2   5        g )Nr  r   r  r   )
r   r   r   r   r   r   r   r   r   r   r0   r  r   r   r   rQ   )r5   r   r   r   r+  r!   )	rL   y_pred_pos_not_cancery_pred_pos_cancerfpr_pos_cancerfnr_pos_cancerth_pos_cancerfpr_pos_not_cancerfnr_pos_not_cancerth_pos_not_cancers	            rM   test_det_curve_pos_labelr    s    Z!^|nq00FHH%WX114=51NM
 AJA=,= v}}S1111Q6==#5555 Ntt$<=Ntt$<=rO   c                    [        U " SS//SS//5      S5        [        U " SS//SS//5      S5        [        U " SS//SS//5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / S	Q// SQ/5      S5        [        U " / S
Q// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / S	Q// SQ/5      S5        [        U " / S
Q// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / S	Q// SQ/5      S5        [        U " / S
Q// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " SS//SS//5      S5        [        U " SS//SS//5      S5        [        U " SS//SS//5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / S	Q// SQ/5      S5        [        U " / S
Q// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        [        U " / SQ// SQ/5      S5        g )Nr   r0   r   r   r   r   r  r   r   r   gUUUUUU?r   g?r   g?r   r   r   r  r   r   r   r  )r   r   r   UUUUUU?)r0   r0   r0   r   )r   r   r   r   )r"   
lrap_scores    rM   check_lrap_toyr    sp   
QF8tTl^<a@
QF8tTl^<eD
QF8tTl^<a@
I;1B0CDaH
I;1B0CDeL
I;1B0CDaH
I;1B0CDeLI;!2 346I I;!2 346I 
I;1B0CDeL
I;1B0CDeLI;!2 346I 
I;1B0CDaH
I;1B0CDoV
I;1B0CDaH
I;1B0CDaH
I;1B0CDeL
I;1B0CDaH
I;1B0CDoV
I;1B0CDeLI;!2 346I 
I;1B0CDaH
I;1B0CDaH 
QF8sCj\:C@
QF8sCj\:C@
QF8sCj\:A>
I;1A0BCSI
I;1A0BCSI
I;1A0BCQG
I;1A0BCUKI;!1 235H I;!1 235H 
I;1A0BCQG
I;0ABEJ
L>4H3IJERrO   c           	         [        S5      n[        SS5       H  nUR                  SU4S9n[        R                  " U5      n[        R
                  " SU45      nU " XS5      S:X  d   eU " XT5      S:X  d   e[        R                  " SU45      nU " XS5      S:X  d   eU " XT5      S:X  a  M   e   [        U " S/S/S/S//S/S/S/S//5      S5        g )Nr   r+   r   r0   r   r^   r   )r)   r`   re  r5   r  r   r   r"   )r  r/   n_labelsrJ   y_score_tiesrL   s         rM   !check_zero_or_all_relevant_labelsr  *  s    %a(L!QK&&QM&:}}W- 1h-(&*c111&/3666 !X'&*c111&/3666   QC!qcA3'3%#u)EFrO   c                    [         R                  " [        5         U " / SQ/ SQ5        S S S 5        [         R                  " [        5         U " / SQ/ SQ/ SQ/ SQ/5        S S S 5        [         R                  " [        5         U " / SQ/ SQ/ SQ/ SQ/5        S S S 5        [         R                  " [        5         U " SS/SS//SS/5        S S S 5        [         R                  " [        5         U " SS/SS//SS//5        S S S 5        [         R                  " [        5         U " SS/SS//S/S//5        S S S 5        [         R                  " [        5         U " SS//SS/SS//5        S S S 5        [         R                  " [        5         U " S/S//SS/SS//5        S S S 5        [         R                  " [        5         U " SS/SS//S/S//5        S S S 5        g ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNt= f! , (       d  f       GNR= f! , (       d  f       GN/= f! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g = f)	Nr   )r   r   r   r   )r   r   r   )r   r   r   )r   r   r   r   r0   )r   r   r   r  s    rM   check_lrap_error_raisedr  A  s   	z	"9./ 
#	z	"90/?ST 
#	z	"//R	
 
# 
z	"QFQF#aV, 
#	z	"QFQF#q!fX. 
#	z	"QFQF#qcA3Z0 
#	z	"QF8q!fq!f-. 
#	z	"QC!:AA/0 
#	z	"QFQF#qcA3Z0 
#	") 
#	"	"	"	"	" 
#	"	"	"	"	"	"	"	"	"	"	"sk   G G2H9H-H("H:III. 
G/2
H
H
H%(
H7:
I	
I
I+.
I<c           	         [        SS5       Hy  n[        R                  " SU45      n[        SU5       HN  n[        X-
  5       H:  n[        R                  " SU45      nSUSXDU-   24'   [	        U " XR5      X1-  5        M<     MP     M{     g )Nr+   rN  r0   r   )r`   r5   r   r   r"   )r  r  rJ   
n_relevantrX   rL   s         rM   check_lrap_only_tiesr  [  s     !RL''1h-(  8,JX231h-045q#j 0001#Jv$?AVW 4 -	 !rO   c                   ^^ [        SS5       H  nU[        R                  " U5      R                  SU45      S-   -
  n[        R                  " SU45      nSUS'   SUS'   [        U " X25      SU-  S-   S-  5        [        SU5       Hk  m[        UT-
  5       HV  m[        R                  " SU45      nSUSTTT-   24'   [        U " X25      [        UU4S j[        T5       5       5      5        MX     Mm     M     g )Nr+   rN  r0   )r   r   )r   rQ   r   c              3   F   >#    U  H  nUS -   TU-   S -   T-  -  v   M     g7f)r0   Nr  ).0r  r  rX   s     rM   	<genexpr>>check_lrap_without_tie_and_increasing_score.<locals>.<genexpr>~  s.      !2A QC!GaK:#=>!2s   !)r`   r5   r6   rS   r   r"   rT   )r  r  rJ   rL   r  rX   s       @@rM   +check_lrap_without_tie_and_increasing_scorer  j  s     !RLbii1991h-H1LM 1h-(tuJv7!h,:Ja9OP  8,JX
231h-045q#j 0001#v/ !&z!2  4 - !rO   c                   ^^ [        X5        [        U 5      n [        U5      nU R                  u  p#[        R                  " U45      n[        U5       H  n[        R                  " X   SS9u  pgUR                  nX-
  m[        R                  " TUS-   S9R                  5       n	U	T   mX   R                  5       S   n
U
R                  S:X  d  U
R                  U:X  a  SXE'   M  SXE'   U
 H+  m[        UU4S jU
 5       5      nXE==   UTT   -  -  ss'   M-     XE==   U
R                  -  ss'   M     UR                  5       $ )z8Simple implementation of label ranking average precisionT)return_inverser0   )	minlengthr   r   c              3   :   >#    U  H  nTU   TT   :*  v   M     g 7fr  r  )r  r  labelranks     rM   r  _my_lrap.<locals>.<genexpr>  s      JAaDK!7s   )r(   r'   r4   r5   emptyr`   rR   r   bincountcumsumnonzerorT   r  )rL   rJ   rD   r  rc   rd   unique_rankinv_rankn_ranks	corr_rankrelevantn_ranked_abover  r  s               @@rM   _my_lrapr    s,   F, F'"G ,,IHHi\"E9 !#		'*T J""! KK!<CCE	9$$&q)==A(!:EHE ! J JJN He44H  	HMM!5 8 ::<rO   r   c                 h   [        SSUUUS9u  pE[        UR                  S   UR                  S   US9n[        US5      (       a  UR	                  5       n[        XV5      n[        XV5      n[        Xx5        [        U5      nUR                  X!4S9n[        XV5      n[        XV5      n[        Xx5        g )Nr0   F)rE   allow_unlabeledr/   	n_classesrD   r   )n_componentsrE   r/   toarrayr   )
r   r   r4   hasattrr
  r   r  r"   r)   re  )	r  r  rD   r/   ry   rL   rJ   
score_lrapscore_my_lraps	            rM   %check_alternative_lrap_implementationr    s     /!IA $\\!_<<?!G w	""//#6vGJV-M
2 &l3L""(>"?G6vGJV-M
2rO   checkfuncc                     U " U5        g r  r  )r  r  s     rM   test_label_ranking_avpr    s     
$KrO   c                  "    [        [        5        g r  )r  r   r  rO   rM   test_lrap_error_raisedr    s    ABrO   rD   )r0   r+   r   rH  r  )r+   r   rN  r/   c                 &    [        [        XU5        g r  )r  r   )rD   r  r/   s      rM   $test_alternative_lrap_implementationr    s     *-y\rO   c                  h   [         R                  " / SQ/ SQ/ SQ/[        S9n [         R                  " / SQ/ SQ/ SQ/5      n[         R                  " / SQ5      n[         R                  " / S	Q5      n[        [	        XUS
9[         R
                  " X2-  5      [         R
                  " U5      -  5        g )Nr  r  r  rO  )r   r   r   r   )r   r   r   r   r   r   r   r   )r   r   r^   )r^   r^   r   r   )r5   r   boolr"   r   rT   )rL   rJ   samplewise_lrapsr   s       rM   &test_lrap_sample_weighting_zero_labelsr    s     XX|\<@MFhh	35IJG xx 01HH_-M-=	
 	}/0266-3HH	rO   c            	         [        [        SS//SS//5      S5        [        [        SS//SS//5      S5        [        [        SS//SS//5      S5        [        [        SS//SS//5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / S	Q// SQ/5      S5        [        [        / S
Q// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / S	Q// SQ/5      S5        [        [        / S
Q// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / S	Q// SQ/5      S5        [        [        / S
Q// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / S	Q/ SQ// SQ/ SQ/5      S5        [        [        / S	Q/ SQ/ S
Q// SQ/ SQ/ SQ/5      S5        [        [        / S	Q/ SQ/ S
Q// SQ/ SQ/ SQ/5      S5        g )Nr   r0   r   r   r+   r  r  r   r   r   r   r   r   r   r  r  r  r   g      $@r   r0   r   r1  r   rN  r  rA  g@r   r0   r   r"   r   r  rO   rM   test_coverage_errorr#    s   Ax4,@!DAx4,@!DAx4,@!DAx4,@!D	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L 	9-/KL
 	9-y)/T	
 		 	9-y)/T	
 		rO   c                     [        [        SS//SS//5      S5        [        [        SS//SS//5      S5        [        [        SS//SS//5      S5        [        [        SS//SS//5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / S	Q// SQ/5      S5        [        [        / S
Q// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        g )Nr   r   r0   r+   r  r  r   r   r   r   r   r   r   r  r"  r  rO   rM   test_coverage_tie_handlingr%  2  s.   Ax3*>BAx3*>BAx3*>BAx3*>B	{5E4FGK	{5E4FGK	{5E4FGK	{5E4FGK	{5E4FGK	{5E4FGK	{5E4FGK	{5E4FGKrO   zy_true, y_scorec                     [         R                  " [        SS9   [        X5        S S S 5        g ! , (       d  f       g = f)Nz'Expected 2D array, got 1D array insteadr   )r   r   r   r   rU  s     rM   test_coverage_1d_error_messager'  B  s(     
z)S	Tv' 
U	T	Tr  c            	      h   [        [        SS//SS//5      S5        [        [        SS//SS//5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / S	Q// SQ/5      S5        [        [        / S
Q// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        SS//SS//5      S5        [        [        SS//SS//5      S5        [        [        SS//SS//5      S5        [        [        SS//SS//5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ/ SQ// SQ/ SQ/5      S5        [        [        / SQ/ SQ/ S	Q// SQ/ SQ/ SQ/5      S5        [        [        / SQ/ SQ/ S	Q// SQ/ SQ/ SQ/5      S5        g )Nr   r0   r   r   r   r  r   r   r   r   r^   r   r   r  r  r  r  r  r  r   rA  r!  r"   r   r  rO   rM   test_label_ranking_lossr*  Q  s   *QF8tTl^DaH*QF8tTl^DaH*I;9J8KLaP*I;9J8KLeT*I;9J8KLaP*I;9J8KLeT*I;9J8KLeT*I;9J8KLeT *QF8tTl^DaH*QF8tTl^DaH*QF8sCj\BAF*QF8sCj\BAF*I;9J8KLaP*I;9J8KLaP*I;9I8JKQO*I;9I8JKQO Iy1OY3OP
 	9-y)/T	
 	"	 	9-y)/T	
 	"	rO   csr_containerc                 |    [        [        U " [        R                  " / SQ/ SQ/5      5      / SQ/ SQ/5      S5        g )Nr   r   r   r!  r   )r"   r   r5   r   )r+  s    rM   test_label_ranking_loss_sparser-  |  s4    "((Iy#9:;mY=W	
 		rO   c                  x   [         R                  " [        5         [        SS/SS//SS/5        S S S 5        [         R                  " [        5         [        SS/SS//SS//5        S S S 5        [         R                  " [        5         [        SS/SS//S/S//5        S S S 5        [         R                  " [        5         [        SS//SS/SS//5        S S S 5        [         R                  " [        5         [        S/S//SS/SS//5        S S S 5        [         R                  " [        5         [        SS/SS//S/S//5        S S S 5        g ! , (       d  f       GN+= f! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g = f)Nr   r0   )r   r   r   r   r  rO   rM   $test_ranking_appropriate_input_shaper/    sI   	z	"QFQF+aV4 
#	z	"QFQF+q!fX6 
#	z	"QFQF+qcA3Z8 
#	z	"QF8q!fq!f%56 
#	z	"QC!:AA'78 
#	z	"QFQF+qcA3Z8 
#	" 
#	"	"	"	"	"	"	"	"	"	"	"sG   EE&
E8F	;F4F+
E#&
E58
F	
F
F(+
F9c                     [        [        SS//SS//5      S5        [        [        SS//SS//5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / SQ// SQ/5      S5        [        [        / S	Q// SQ/5      S5        [        [        / S
Q// SQ/5      S5        g )Nr0   r   r   r   r  r   r   r   r   r   r)  r  rO   rM   test_ranking_loss_ties_handlingr1    s    *QF8sCj\BAF*QF8sCj\BAF*I;9I8JKUS*I;9I8JKUS*I;9I8JKQO*I;9I8JKQO*I;9I8JKQO*I;9I8JKQOrO   c                      [        SSS9u  pU* S-   n[        X5        [        R                  R	                  S5      R                  S5      u  p[        X5        g Nr   rN  )r/   r  r0   )r+   r   rN  )r   _test_dcg_score_forr5   r9   r:   random_samplery   rL   rJ   s      rM   test_dcg_scorer7    sL    .ALIAgkG(ii++A.<<\JOF(rO   c                 .   [         R                  " [         R                  " U R                  S   5      S-   5      n[	        X 5      n[	        X5      nXC:*  R                  5       (       d   e[	        X SS9U:*  R                  5       (       d   eUR                  U R                  S   4:X  d   eUR                  U R                  S   4:X  d   eU[        R                  " [         R                  " U 5      S S 2S S S24   U-  R                  SS95      :X  d   eg )Nr0   r+   r   kr   rQ   axis)
r5   log2r6   r4   r   r`  r   r+  sortrT   )rL   rJ   discountidealrc   s        rM   r4  r4    s    wwryya1A56Hv.Ev/EN!!!!v3u<AACCCC;;6<<?,,,,;;6<<?,,,,FMM2776?1dd7#;h#F"K"KQR"K"STTTTrO   c            	         [         R                  " [         R                  " S5      /5      n [         R                  " U R                  5      n[        X5      n[        XSS9nS[         R                  " [         R                  " SS5      5      -  nU[        R                  " UR                  5       U R                  5       -  /5      :X  d   eU[        R                  " X@S S 2S S S24   -  R                  5       /5      :X  d   eSUSS	S 24'   [        X5      n[        XSS9nU[        R                  " X@S S 2S S S24   -  R                  5       /5      :X  d   eU[        R                  " US S R                  5       U SS	S 24   R                  5       -  USS  R                  5       U SS S	24   R                  5       -  -   /5      :X  d   eg )
Nr   Tignore_tiesr0   r+   r   rQ   r   r   )r5   r7  r6   r   r4   r   r=  r   r+  rT   r  )rL   rJ   dcgdcg_ignore_ties	discountss        rM   test_dcg_tiesrG    s   ZZ1'Fhhv||$G
V
-C(dKOBGGBIIaO,,I&--6;;=!@ ABBBBfmmiDbD/.I-N-N-P,QRRRRGAqrEN
V
-C(dKOfmmiDbD/.I-N-N-P,QRRRR&--bqM&AB-"4"4"66m!F1bqb5M$6$6$889	
   rO   c            
          [         R                  " S5      R                  S5      n [        X SSS9[        R
                  " [        X SSS95      :X  d   eg )N   )r+   r   r   T)r:  rC  )r5   r6   rS   r   r   r+  )r   s    rM   test_ndcg_ignore_ties_with_krJ    sL    
		"f%AaaT2fmm11$/7   rO   c                      [         R                  " / SQ/5      n [         R                  " / SQ/5      nSn[        R                  " [        US9   [        X5        SSS5        g! , (       d  f       g= f)zDCheck `ndcg_score` exception when `y_true` contains negative values.)g{Gzg(\gGz޿g(\?gQ?)gQ?gףp=
?r   gQ?gHzG?z7ndcg_score should not be used on negative y_true valuesr   N)r5   r   r   r   r   r   )rL   rJ   r   s      rM    test_ndcg_negative_ndarray_errorrL    sO    XX89:Fhh678GP	z)9	:6# 
;	:	:s   A##
A1c                     [         R                  " S5      R                  SS5      n U [         R                  R	                  S5      R                  SSU R                  S9-   n[        X5      n[        XSS	9nU[        R                  " U5      :X  d   eU[        R                  " S
5      :X  d   eUS-  n[        X5      [        R                  " S
5      :X  d   eg )NF   r   rN  r   皙ɿr   r   TrB  r^   r2  )
r5   r6   rS   r9   r:   re  r4   r   r   r+  )rL   rJ   ndcgndcg_no_tiess       rM   test_ndcg_invariantrR    s    YYr]""1b)Fryy,,Q/77c7UUGf&Df4@L6==....6==%%%%tOGf&&--*<<<<rO   rC  c           
      &   S[         R                  " S5      S S -  n[         R                  " [         R                  " SSS5      S5      nU[         R                  R                  S5      R                  SS	UR                  S
9-   n[        XU S9[        R                  " S[         R                  " [         R                  " SS5      5      -  5      :X  d   e[        XU S9[        R                  " S[         R                  " [         R                  " SS5      5      -  5      :X  d   e[        XU S9[        R                  " S[         R                  " [         R                  " SS5      5      -  5      :X  d   e[        XSU S9[        R                  " S[         R                  " [         R                  " SS5      5      -  5      :X  d   e[        XU S9[        R                  " S[         R                  " [         R                  " SS5      5      -  R                  5       5      :X  d   e[!        XU S9[        R                  " S[         R                  " [         R                  " SS5      5      -  R                  5       5      :X  d   eS[         R"                  " S5      -  nS[         R                  " [         R                  " SS5      5      -  R%                  5       n[        XU S9[        R                  " U[         R"                  " S5      -  5      :X  d   e[        XU S9[        R                  " [         R"                  " S5      5      :X  d   e[!        XU S9[        R                  " U5      :X  d   e[        XU S9[        R                  " S5      :X  d   eg )Nr   r   r   r   rQ   )r   r0   r   rO  r   r   rB  r+   r0   rN  )log_baserC  )r   r   	   r^   )r5   eyetiler6   r9   r:   re  r4   r   r   r+  r=  r   log10r   r  r   r   rT   )rC  rL   rJ   y_score_noisyexpected_dcg_scores        rM   test_ndcg_toy_examplesr[    s   2AFggbii2r*F3Gbii33A6>>c ?  M [	q277299Q?33	45 5 5 ;	q277299Q?33	45 5 5 [	q277299Q?33	45 5 5 "+	q288BIIaO44	56 6 6 f;?6==	
RWWRYYq!_%	%++-D    V+>&--	
RWWRYYq!_%	%++-C     Fbggbii1o66;;=[	)BGGAJ6	78 8 8 [	rwwqz	"# # # V+>&--C    f;?6==QTCUUUUrO   c                      Sn [         R                  " [        U S9   [        S//S//5        SSS5        g! , (       d  f       g= f)zdCheck that we raise an informative error message when trying to
compute NDCG with a single document.zTComputing NDCG is only meaningful when there is more than 1 document. Got 1 instead.r   r0   N)r   r   r   r   )r  s    rM   test_ndcg_error_single_documentr]    s:    	  
z	1QC5A3%  
2	1	1s	   6
Ac                      [        SSS9u  pU* S-   n[        X5        [        R                  R	                  S5      R                  S5      u  p[        X5        g r3  )r   _test_ndcg_score_forr5   r9   r:   r5  r6  s      rM   test_ndcg_scorer`    sL    .ALIAgkG)ii++A.<<\JOF)rO   c                    [        X 5      n[        X5      nX2:*  R                  5       (       d   eU S:H  R                  SS9nX$)    [        R                  " [        R
                  " U) R                  5       5      5      :X  d   eX$   [        R                  " [        R                  " UR                  5       5      5      :X  d   eX4)    [        R                  " [        X5      U)    [        X 5      U)    -  5      :X  d   eX4   [        R                  " [        R                  " UR                  5       5      5      :X  d   eUR                  U R                  S   4:X  d   eUR                  U R                  S   4:X  d   eg )Nr   r0   r;  )
r   r`  r   r+  r5   r   rT   r   r   r4   )rL   rJ   r@  rc   all_zeros        rM   r_  r_    sC   /E0EN!!!!!  a (Hv}}RWWxi__5F-GHHHH?fmmBHHX\\^,DEEEEv}}6+XI6
V
,hY
7	8     ?fmmBHHX\\^,DEEEE;;6<<?,,,,;;6<<?,,,,rO   c            	      4   [         R                  " / SQ5      n [        X SS9S:X  d   e[        X SS9S:X  d   e[        R                  " [
        5         [        X SS9(       d   e S S S 5        [        R                  " [
        5         [        X SS9(       d   e S S S 5        [        R                  " [
        5         [        X SS9(       d   e S S S 5        [         R                  " / SQ5      n[        XSS9n[        X5      nX#:X  d   e[        XS	S9S
:X  d   e[        SS9u  pn[         R                  " SSS5       H!  n[        [        XUS9[        XU5      5        M#     g ! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       N= f)Nr   r0   )rv   gMbP?gg?r   )r   r   r   g{Gz?r   r   Tr   g-C6?r   )
r5   r   r   r   r   r   rN   linspacer"   r   )rL   r  roc_auc_with_max_fpr_oneunconstrained_roc_aucrK   ry   rv   s          rM   test_partial_roc_auc_scorerg  .  s@   XXl#F3q88871<<<	z	"VT::: 
#	z	"VS999 
#	z	"VQ777 
# xx+,H,VqI)&;#<<<373>>>'t4FA;;tQ*&':"67;	
 + 
#	"	"	"	"	"s$   E&E8=F	&
E58
F	
Fzy_true, k, true_scorer   r+   r   r   c                     [         R                  " / SQ/ SQ/ SQ/ SQ/5      n[        XUS9nU[        R                  " U5      :X  d   eg )Nr  r   r   r   r   r   r   r   r   r   r   r   r   r9  r5   r   r   r   r+  )rL   r:  
true_scorerJ   rc   s        rM   test_top_k_accuracy_scorern  H  sH     hh    		
G !A6EFMM*----rO   zy_score, k, true_score)rQ   rQ   r0   r0   )rQ   r0   rQ   r0   )r   r   r   r   )r   r   r   r   c                 *   / SQnU R                  5       S:  a  U R                  5       S::  a  SOSnUS:X  a!  X:  R                  [        R                  5      OUn[        X0US9n[        X55      nXgs=:X  a  [        R                  " U5      :X  d   e   eg )Nr   r   r0   r   r9  )	minmaxastyper5   int64r   r   r   r+  )rJ   r:  rm  rL   rm   rK   rc   	score_accs           rM    test_top_k_accuracy_score_binaryru  ]  s     F{{})gkkmq.@aI78Avg!))"((36F A6Ev.I:z!::::::rO   zy_true, true_score, labels)r   r0   r0   r+   r  )r   rB  rB  r   )r   r   r  rB  labels_as_ndarrayc                     U(       a  [         R                  " U5      n[         R                  " / SQ/ SQ/ SQ/ SQ/5      n[        XSUS9nU[        R
                  " U5      :X  d   eg)z,Test when labels and y_score are multiclass.r  ri  rj  rk  r+   r:  r  N)r5   r7  r   r   r   r+  )rL   rm  r  rv  rJ   rc   s         rM   0test_top_k_accuracy_score_multiclass_with_labelsry  t  s[     F#hh    		
G !AfEEFMM*----rO   c            
         [         R                  " SSSSS9u  p[        XSS9u  p#pE[        SS9nUR	                  X$5        [        X#4XE45       Hl  u  p[        SS5       Vs/ s H  n[        XR                  U 5      US9PM     nn[        R                  " [        R                  " U5      S:  5      (       a  Ml   e   g s  snf )NrN  r2  r   )r  rD   n_informativer/   r.   r+   r9  )r   make_classificationr   r   r=   rr  r`   r   r>   r5   r`  r   )	rB   rC   X_trainX_testy_trainy_testrI   r:  scoress	            rM   $test_top_k_accuracy_score_increasingr    s    ''BQDA (81'M$GW
!
,CGGGW%'89HMaQS
HT1 $5$5a$8A> 	 
 vvbggfo)**** :
s   "#C c                     [         R                  " / SQ/ SQ/ SQ/ SQ/5      n[        XUS9[        R                  " U5      :X  d   eg )N)r   r   r   r   )r0   r   r   r   )r   r   r   r   r  r9  rl  )rL   r:  rm  rJ   s       rM   test_top_k_accuracy_score_tiesr    sC     hh		
G  15z9RRRRrO   z	y_true, kr   c                     [         R                  " / SQ/ SQ/ SQ/ SQ/5      nSn[        R                  " [        US9   [        XUS9nS S S 5        WS:X  d   eg ! , (       d  f       N= f)	Nr  )r   r   r   r   )r   r   r   r   )r   r   r   r   zu'k' \(\d+\) greater than or equal to 'n_classes' \(\d+\) will result in a perfect score and is therefore meaningless.r   r9  r0   )r5   r   r   r   r   r   )rL   r:  rJ   r   rc   s        rM   !test_top_k_accuracy_score_warningr    sg     hh    		
G	6  
,4D	E$V: 
FA:: 
F	Es   A
A(zy_true, y_score, labels, msg)r   g=
ףp=?r0   r+   )r   r   r   )r   r   r   r   )r   r   r   z9y type must be 'binary' or 'multiclass', got 'continuous'zZNumber of classes in 'y_true' \(4\) not equal to the number of classes in 'y_score' \(3\).)r   r   r   r   z"Parameter 'labels' must be unique.z#Parameter 'labels' must be ordered.)r   r   r0   r+   zSNumber of given labels \(4\) not equal to the number of classes in 'y_score' \(3\).r  z3'y_true' contains labels not in parameter 'labels'.)r   r   r   )r   r   r   z}`y_true` is binary while y_score is 2d with 3 classes. If `y_true` does not contain all the labels, `labels` must be providedc                     [         R                  " [        US9   [        XSUS9  S S S 5        g ! , (       d  f       g = f)Nr   r+   rx  )r   r   r   r   )rL   rJ   r  rD  s       rM   test_top_k_accuracy_score_errorr    s+    f 
z	-V&A 
.	-	-r  c                     U " / SQ/ SQ/5      n[         R                  " / SQ/ SQ/5      n[        X5      nU[        R                  " S5      :X  d   eg )Nr   r   )r   r   r   r  )r5   r   r   r   r+  )r+  rL   rJ   results       rM   Otest_label_ranking_avg_precision_score_should_allow_csr_matrix_for_y_true_inputr  )	  sG     Iy12Fhh34G26CFV]]5))))rO   metricr4  ))FT)r   r0   )r   r^   )zeroonec                    [         R                  R                  S5      nSUS   pCUR                  XSS9nUR	                  U5      nU " XVUS9n[        U[        5      (       a  [         R                  " U5      (       a   egUu  pn
[         R                  " U5      R                  5       (       a   e[         R                  " U	5      R                  5       (       a   e[         R                  " U
5      R                  5       (       a   eg)zCheck that the metric works with different types of `pos_label`.

We can expect `pos_label` to be a bool, an integer, a float, a string.
No error should be raised for those types.
*   rN  rQ   T)r   replacer  N)	r5   r9   r:   choicerI  
isinstancerU   r   any)r  r4  rG   rD   rW   rL   y_probar  metric_1metric_2r   s              rM   #test_ranking_metric_pos_label_typesr  5	  s     ))


#Cwr{yZZZ>Fhhy!GFy9F&%  88F#####)/&J88H%))++++88H%))++++88J'++-----rO   c                     [         R                  R                  U 5      nUR                  SSSS9nUR	                  S5      n[        X#5      u    pE[         R                  " US   5      (       d   eg)zCheck that thresholds do not exceed 1.0 when `y_score` is a probability
estimate.

Non-regression test for:
https://github.com/scikit-learn/scikit-learn/issues/26193
r   r+   rN  r   N)r5   r9   r:   r   rI  r   isinf)r:  rG   rL   rJ   ry   r   s         rM   )test_roc_curve_with_probability_estimatesr  O	  sb     ))

 2
3C[[AB['FhhrlG 1Aq88JqM""""rO   c                     Uc  [         R                  " U5      n[         R                  " X-  5      n[         R                  " X-  5      nX44$ )z0Dummy metric that returns a tuple of two values.)r5   	ones_likerT   )rL   rK   r   
y_pred_sum
y_true_sums        rM   _dummy_metricr  ]	  sA    V,./J./J##rO   metric_funcr   r0   r+   r0   r   r+   c           
         [         R                  " / SQ5      n[         R                  " / SQ5      n[        X#XS9u  pE/ nU H0  nX7:  R                  [        5      nUR                  U " X(US95        M2     [        U5      [        U5      :X  d   e[         R                  " [         R                  " U5      S:*  5      (       d   eUb@  [        U[         R                  " [         R                  " X1S:g     5      5      SSS2   5        O:[        U[         R                  " [         R                  " U5      5      SSS2   5        [        XF5        g)z,Test `metric_at_thresholds` outputs correct.r   r   r   r   r   r   r   r   NrQ   )r5   r   r   rr  r8   rr   rV   r`  r   r!   r>  rR   )	r  r   rL   rJ   metric_valuesr   expected_scoresrm   rK   s	            rM   test_metric_at_thresholdsr  f	  s    XXo&Fhh01G 4!M
 O	&..s3{6WX   }Z00066"''*%*++++ 		'12D*E FG"M	
 	
BGGBIIg,>$?"$EFM3rO   c                 Z    [         R                  " U5      [         R                  " U 5      4$ )z2Dummy metric that does not accept `sample_weight`.)r5   rT   )rL   rK   s     rM   _dummy_metric_no_sample_weightr  	  s    FF6NBFF6N++rO   c                     [         R                  " / SQ5      n [         R                  " / SQ5      n[         R                  " / SQ5      n[        R                  " [        SS9   [        X[        US9u    nSSS5        g! , (       d  f       g= f)zLTest `TypeError` is raised when `metric_func` does not take `sample_weight`.r   r   r   r   r   r   r0   r+   r   r0   r+   z"got an unexpected keyword argumentr   r   N)r5   r   r   r   r}  r   r  )rL   rJ   r   ry   s       rM   -test_metric_at_thresholds_sample_weight_errorr  	  s_    XXo&Fhh12GHH_-M	y(L	M#;=
1 
N	M	Ms   "A>>
B	normalizec           	      r   [         R                  " / SQ5      n[         R                  " / SQ5      n[        X[        SU 0S9u  p4/ nU H3  nX&:  R	                  [
        5      nUR                  [        XU S95        M5     [        X55        [        U5      [        [         R                  " U5      5      :X  d   eg)z@Test `metric_params` passed correctly to `metric_at_thresholds`.r   r  r  metric_params)r  N)
r5   r   r   r   rr  r8   rr   r!   rV   rR   )r  rL   rJ   r  r   expected_valuesrm   rK   s           rM   'test_metric_at_thresholds_metric_paramsr  	  s     XXo&Fhh12G 4Y7O!M O	&..s3~f	RS   M3z?c"))G"45555rO   rW   c           
         [         R                  " / SQ5      n[         R                  " / SQ5      n[        UU[        U SS.S9u  p4/ nU H4  nX&:  R	                  [
        5      nUR                  [        XU SS95        M6     [        X55        g)z?Test `pos_label` is passed correctly to `metric_at_thresholds`.r   r  r   )rW   zero_divisionr  N)r5   r   r   r   rr  r8   rr   r!   )rW   rL   rJ   r  r   r  rm   rK   s           rM   #test_metric_at_thresholds_pos_labelr  	  s     XXo&Fhh01G 4$-B	!M O	&..s3FiqQ	
   M3rO   c                  f   [         R                  " / SQ5      n [         R                  " / SQ5      n[         R                  " / SQ5      n[        S5      nUR                  [	        U5      5      nX   nX   nX$   n[        X[        US9u  p[        XV[        US9u  p[        X5        [        X5        g)z<Test `y_score` order does not effect `metric_at_thresholds`.r   )r   r   r   r   r   r  r  r   N)r5   r   r)   permutationrV   r   r   r!   )rL   rJ   r   rG   perm_indicesy_true_permy_score_permsample_weight_permr  thresh_1r  thresh_2s               rM   'test_metric_at_thresholds_y_score_orderr  	  s    XXo&Fhh01GHH_-M
R
 C??3w<0L&K(L&4-}H .>ASH H'H'rO   c                  @   [         R                  " / SQ5      n [         R                  " / SQ5      n[         R                  " / SQ5      n[        X[        5      u  p4[        X [        5      u  pV[	        XF5        [	        X55        [         R                  " / SQ5      n[         R                  " / SQ5      n[        X[        US9u  p4[        X[        US9u  pV[	        XF5        [
        R                  " [        5         [	        X55        SSS5        g! , (       d  f       g= f)a  Test duplicate `y_score` edge cases in `metric_at_thresholds`.

If there are duplicate `y_score` values and `y_true` differs between
these duplicate values, `y_score` order will not affect metric output.

However, if there are duplicate `y_score` values and `sample_weight` differs
between these duplicate values, `y_score` order can affect metric output,
as stable sort preserves relative order.
)r   r   r   r   r   )r0   r   r0   r0   r   r   r  )r+   r+   r0   r   r0   r   N)r5   r   r   r   r!   r   r   AssertionError)	rJ   y_true_1y_true_2r  r  r  r  sample_weight_1sample_weight_2s	            rM   9test_metric_at_thresholds_y_score_order_duplicate_y_scorer  	  s     hh01Gxx(Hxx(H-hPH-hPHH'H' hh/Ohh/O->H .>H
 H'	~	&+ 
'	&	&s   :D
Dc                  d   [         R                  " / SQ5      n [         R                  " / SQ5      n[        U U5      u  p#pEn[        X[        SSS/0S9u  px[        X5        [        USS2SS4   U5        [        USS2SS4   U5        [        USS2SS4   U5        [        USS2SS4   U5        g)zTest `metric_at_thresholds` consistency with `confusion_matrix_at_thresholds`.

This also checks output when `metric_func` returns a tuple of arrays.
r   )r   r   r   r   r   r  r   r0   r  N)r5   r   r   r   r   r$   )	rL   rJ   rg  rh  ri  rj  thresholds_cmr  r   s	            rM   ;test_metric_at_thresholds_consistency_with_confusion_matrixr  
  s    
 XXo&Fhh01G(F)%Cc
 !5)(QF9K!M z1 }Q1W-s3}Q1W-s3}Q1W-s3}Q1W-s3rO   c                     [         R                  " / SQ5      n [         R                  " / SQ5      n[        X[        S[         R                  0S9u  p#[         R
                  " [         R                  " U5      5      (       d   eg)z,Test `metric_at_thresholds` with NaN output.)r   r   r   r   r   r  r  r  N)r5   r   r   r   r   r`  r   )rL   rJ   r  ry   s       rM   *test_metric_at_thresholds_with_nan_outputsr  "
  s^     XXo&Fhh01G+orvv5NM 66"((=)****rO   c                      [         R                  " / SQ5      n [         R                  " / SQ5      n[        R                  " [        SS9   [        XS 5        S S S 5        g ! , (       d  f       g = f)Nrp  )r   r   r   r   z"Pass pos_label=None as keyword argr   )r5   r   r   r   FutureWarningr   rU  s     rM   ?test_confusion_matrix_at_thresholds_positional_args_deprecationr  0
  sE    XXl#Fhh+,G	m+O	P&v= 
Q	P	Ps   
A  
A.)NF)r   rH  r   r  )r   r   numpyr5   r   scipyr   sklearnr   sklearn.datasetsr   sklearn.exceptionsr   sklearn.linear_modelr   sklearn.metricsr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   sklearn.metrics._rankingr   r   r   sklearn.model_selectionr   sklearn.preprocessingr   sklearn.random_projectionr   sklearn.utils._testingr    r!   r"   r#   r$   sklearn.utils.extmathr%   sklearn.utils.fixesr&   sklearn.utils.validationr'   r(   r)   CURVE_FUNCSrN   r\   rg   ro   r   markparametrizer   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r'  r/  r@  rE  rL  rR  rV  r^  rc  rk  rn  rx  filterwarningsr~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r`   r  r  r#  r%  r'  r*  r-  r/  r1  r7  r4  rG  rJ  rL  rR  r[  r]  r`  r_  rg  rn  ru  ry  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rO   rM   <module>r     s    	     ; 5 3    ( 
 5 0 ;  * .  #	)#`2<&H8 $/
) 0
)
")&#))8)6oNdM	).&" 	,	+	,	&	/	t$	/X/Xd 	,	+	&	'9: 	,	&	t$	y!	/	D D:( 	,bhh|.DiP/HH)*	
 HH\"	
 1HH)*	
 HH\"F	
 HH)*#J	
 HH\"	
 HH)* 	
 AHH)*	
 AHH)*	
 AHH\"	
}CFN 7P 8OFPP ' "%8	
' "%8	
F "B7	
 "c2	
 
4R8="F	0G"F	0&*L"
D686& {3# 4# {3E 4E0 LM(>	'JK(9:' ; L N' {3	& 4	& $/  0 >), $/` 0`F6D<=
=>*.$2 .	K!qc*	NQC!-	NQC!-	%sQC0	K	:	NM9=	NM9=	%}i@	K#,	NQC!-	NQC!-	%sQC0	K>	NNLA	NNLA	%~|D	KM:	NI}=	NI}=	%y-@)2'32' N 
K!1<G	Ki>	N$4lEJ	NKDA	KM5A	KM4@	NI}eD	NI}dC'' .
Q#saVaV,
Q#saVaV,	$sAhA7	$sAhA7	$qcA3/	$i;	$i=	$i=	'' $EF2 G2 							 
Q?@	QH?@	K!FG	K!FG.(	

"">48Sv.14X6#N 9:3@ 3)	 "G!RS TC m4j1q2 3 2 5*4nL  	$%	%&'
&'(((V .9 :9 	P)U($= u6#V 7#VL!*- 
4 	q$	q#	q$
.
. 	.	!1a(	.	!1c*	.	!1a(	&	'A.	&	'C0	&	'A.
	;
	;  	,	|4	,	l3	,	l3	&	'/CD	 ,tUm<. =.&+& 	q$	q#	q!
S
S 	q	q$ " 	 G
	
 	 +	
 !	 !0
	
 !	 1
	
 	  	
 	 A
	
 Fo.W	
KNQdBeQdB
 .9* :* &	3I9U C..(#$ 	 4/1J*KL4 M	4:,
	
 tUm46 56$ q!f-4 .4,(.%,P44
+>rO   